From 38ca98b63207484055e8cfbfd9c6faf349d3199f Mon Sep 17 00:00:00 2001 From: Daniel Ziegenberg Date: Fri, 14 Mar 2025 15:14:10 +0100 Subject: [PATCH 1/2] MDL-81437 core_files: Add ad-hoc task to upgrade mimetypes on update Signed-off-by: Daniel Ziegenberg --- .upgradenotes/MDL-81437-2025031413552718.yml | 8 + .../asynchronous_mimetype_upgrade_task.php | 57 ++++++ ...synchronous_mimetype_upgrade_task_test.php | 164 ++++++++++++++++++ lib/db/upgradelib.php | 29 ++++ lib/tests/db/upgradelib_test.php | 33 ++++ 5 files changed, 291 insertions(+) create mode 100644 .upgradenotes/MDL-81437-2025031413552718.yml create mode 100644 files/classes/task/asynchronous_mimetype_upgrade_task.php create mode 100644 files/tests/task/asynchronous_mimetype_upgrade_task_test.php diff --git a/.upgradenotes/MDL-81437-2025031413552718.yml b/.upgradenotes/MDL-81437-2025031413552718.yml new file mode 100644 index 00000000000..ecb9ca066c3 --- /dev/null +++ b/.upgradenotes/MDL-81437-2025031413552718.yml @@ -0,0 +1,8 @@ +issueNumber: MDL-81437 +notes: + core_files: + - message: | + Adds a new ad-hoc task `core_files\task\asynchronous_mimetype_upgrade_task` to upgrade the mimetype of files + asynchronously during core upgrades. The upgradelib also comes with a new utility function + `upgrade_create_async_mimetype_upgrade_task` for creating said ad-hoc task. + type: improved diff --git a/files/classes/task/asynchronous_mimetype_upgrade_task.php b/files/classes/task/asynchronous_mimetype_upgrade_task.php new file mode 100644 index 00000000000..a2439f02f8b --- /dev/null +++ b/files/classes/task/asynchronous_mimetype_upgrade_task.php @@ -0,0 +1,57 @@ +. + +namespace core_files\task; + +use core\task\adhoc_task; + +/** + * Ad-hoc task that performs asynchronous upgrades of a given file type. + * + * This ad-hoc taks is used during core upgrades. + * + * @package core_files + * @copyright 2025 Daniel Ziegenberg + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class asynchronous_mimetype_upgrade_task extends adhoc_task { + + /** + * Run the adhoc task and update the mime type of files. + */ + public function execute(): void { + global $DB; + + // Upgrade mime type for existing files. + $customdata = $this->get_custom_data(); + + foreach ($customdata->extensions as $extension) { + mtrace("Updating mime type for files with extension *.{$extension} to {$customdata->mimetype}"); + + $condition = $DB->sql_like('filename', ":extension", false); + $select = "{$condition} AND mimetype <> :mimetype"; + $params = [ + 'extension' => $DB->sql_like_escape("%.$extension"), + 'mimetype' => $customdata->mimetype, + ]; + + $count = $DB->count_records_select('files', $select, $params); + $DB->set_field_select('files', 'mimetype', $customdata->mimetype, $select, $params); + + mtrace("Updated {$count} files with extension *.{$extension} to {$customdata->mimetype}"); + } + } +} diff --git a/files/tests/task/asynchronous_mimetype_upgrade_task_test.php b/files/tests/task/asynchronous_mimetype_upgrade_task_test.php new file mode 100644 index 00000000000..ef3ca7e683e --- /dev/null +++ b/files/tests/task/asynchronous_mimetype_upgrade_task_test.php @@ -0,0 +1,164 @@ +. + +namespace core_files\task; + +/** + * Tests for the asynchronous mimetype upgrade task. + * + * @package core_files + * @category test + * @copyright 2025 Daniel Ziegenberg + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers \core_files\task\asynchronous_mimetype_upgrade_task::execute + */ +final class asynchronous_mimetype_upgrade_task_test extends \advanced_testcase { + + /** + * Data provider for test_upgrade_mimetype(). + * + * @return array + */ + public static function upgrade_mimetype_provider(): array { + return [ + 'Single file, one extension' => [ + 'files' => [ + 'filename.extension1' => 'type/subtype', + ], + 'mimetype' => 'type/subtype', + 'extensions' => ['extension1'], + ], + 'Single file, one extension, desired extension substring of actual extension' => [ + 'files' => [ + 'filename.extension1andsomemore' => 'text/plain', + ], + 'mimetype' => 'type/subtype', + 'extensions' => ['extension1'], + ], + 'Single file, one extension, filename same as desired extension' => [ + 'files' => [ + 'extension1.bogus' => 'text/plain', + ], + 'mimetype' => 'type/subtype', + 'extensions' => ['extension1'], + ], + 'Multiple files, one extension' => [ + 'files' => [ + 'filename_a.extension1' => 'type/subtype', + 'filename_b.extension1' => 'type/subtype', + ], + 'mimetype' => 'type/subtype', + 'extensions' => ['extension1'], + ], + 'Multiple files, multiple extensions' => [ + 'files' => [ + 'filename_a.extension1' => 'type/subtype', + 'filename_b.extension1' => 'type/subtype', + 'filename_a.extension2' => 'type/subtype', + 'filename_b.extension2' => 'type/subtype', + ], + 'mimetype' => 'type/subtype', + 'extensions' => [ + 'extension1', + 'extension2', + ], + ], + 'Multiple files, multiple extensions, some unrelated' => [ + 'files' => [ + 'filename_a.extension1' => 'type/subtype', + 'filename_b.extension1' => 'type/subtype', + 'filename_a.extension2' => 'type/subtype', + 'filename_b.extension2' => 'type/subtype', + 'filename_c.bogus' => 'text/plain', + 'filename_c.bogus2' => 'text/plain', + ], + 'mimetype' => 'type/subtype', + 'extensions' => [ + 'extension1', + 'extension2', + 'extension3', + ], + ], + ]; + } + + /** + * Test upgrading the mimetype of files. + * + * @dataProvider upgrade_mimetype_provider + * + * @param array $files + * @param string $mimetype + * @param array $extensions + */ + public function test_upgrade_mimetype(array $files, string $mimetype, array $extensions): void { + global $DB; + $this->resetAfterTest(); + + // Create files with different extensions. + $fs = get_file_storage(); + foreach (array_keys($files) as $filename) { + $filerecord = [ + 'contextid' => \core\context\system::instance()->id, + 'component' => 'core', + 'filearea' => 'unittest', + 'itemid' => 0, + 'filepath' => '/', + 'filename' => $filename, + ]; + + $fs->create_file_from_string($filerecord, 'content'); + } + + // Create and run the upgrade task. + $upgardetask = new asynchronous_mimetype_upgrade_task(); + $upgardetask->set_custom_data([ + 'mimetype' => $mimetype, + 'extensions' => $extensions, + ]); + ob_start(); + $upgardetask->execute(); + $output = ob_get_clean(); + + // Check that the task output is correct. + foreach ($extensions as $extension) { + $this->assertStringContainsString( + "Updating mime type for files with extension *.{$extension} to {$mimetype}", + $output + ); + $countfiles = count(array_filter( + array_keys($files), + function ($filename) use ($extension) { + return str_ends_with($filename, $extension); + } + )); + $this->assertStringContainsString( + "Updated {$countfiles} files with extension *.{$extension} to {$mimetype}", + $output + ); + } + + // Check that the mimetype was updated and unrelated files remain untouched. + foreach ($files as $filename => $exptectedmimetype) { + $mimetypedb = $DB->get_field( + table: 'files', + return: 'mimetype', + conditions: ['filename' => $filename] + ); + $this->assertEquals(expected: $exptectedmimetype, actual: $mimetypedb); + } + } +} diff --git a/lib/db/upgradelib.php b/lib/db/upgradelib.php index 84a0e1258e5..91e4616d7d7 100644 --- a/lib/db/upgradelib.php +++ b/lib/db/upgradelib.php @@ -2066,3 +2066,32 @@ function upgrade_add_explain_action_to_ai_providers() { $currentrecords->close(); } + +/** + * Creates a new ad-hoc task to upgrade the mime-type of files asynchronously. + * Thus, we can considerably reduce the time an upgrade takes. + * + * @param string $mimetype the desired mime-type + * @param string[] $extensions a list of file extensions, without the leading dot + * @return void + */ +function upgrade_create_async_mimetype_upgrade_task(string $mimetype, array $extensions): void { + global $DB; + + // Create adhoc task for upgrading of existing files. Due to a code restriction on the upgrade, invoking any core + // functions is not permitted. Thus we craft our own ad-hoc task that will process all existing files. + $record = new \stdClass(); + $record->classname = '\core_files\task\asynchronous_mimetype_upgrade_task'; + $record->component = 'core'; + $record->customdata = json_encode([ + 'mimetype' => $mimetype, + 'extensions' => $extensions, + ]); + + // Next run time based from nextruntime computation in \core\task\manager::queue_adhoc_task(). + $clock = \core\di::get(\core\clock::class); + $nextruntime = $clock->time() - 1; + $record->nextruntime = $nextruntime; + + $DB->insert_record('task_adhoc', $record); +} diff --git a/lib/tests/db/upgradelib_test.php b/lib/tests/db/upgradelib_test.php index 99d1137c9cf..b3f97c757ae 100644 --- a/lib/tests/db/upgradelib_test.php +++ b/lib/tests/db/upgradelib_test.php @@ -540,4 +540,37 @@ final class upgradelib_test extends \advanced_testcase { $this->assertEquals($expected, $block); } } + + /** + * Ensure that the upgrade_create_async_mimetype_upgrade_task function performs as expected. + * + * @covers ::upgrade_create_async_mimetype_upgrade_task + */ + public function test_upgrade_create_async_mimetype_upgrade_task(): void { + global $DB; + $this->resetAfterTest(); + $clock = $this->mock_clock_with_frozen(); + + upgrade_create_async_mimetype_upgrade_task('type/subtype', ['extension1', 'extension2']); + + // Ensure that the task was created with correct next runtime. + $nextruntime = $DB->get_field( + table: 'task_adhoc', + return: 'nextruntime', + conditions: ['component' => 'core', 'classname' => '\core_files\task\asynchronous_mimetype_upgrade_task'], + strictness: MUST_EXIST, + ); + $this->assertEquals(expected: $clock->time() - 1, actual: $nextruntime); + + // Ensure that the task has the correct custom data. + $customdata = $DB->get_field( + table: 'task_adhoc', + return: 'customdata', + conditions: ['component' => 'core', 'classname' => '\core_files\task\asynchronous_mimetype_upgrade_task'], + strictness: MUST_EXIST, + ); + $customdata = json_decode($customdata); + $this->assertEquals(expected: 'type/subtype', actual: $customdata->mimetype); + $this->assertEquals(expected: ['extension1', 'extension2'], actual: $customdata->extensions); + } } From 13743de30b597683bbeb1f9a4bd746a5874f5ae8 Mon Sep 17 00:00:00 2001 From: Daniel Ziegenberg Date: Fri, 14 Mar 2025 15:05:45 +0100 Subject: [PATCH 2/2] MDL-81437 core_files: Adding webp to the standard included MIME types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Luca Bösch --- lib/classes/filetypes.php | 2 ++ lib/db/upgrade.php | 8 ++++++++ version.php | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/classes/filetypes.php b/lib/classes/filetypes.php index 3b0be4b591b..7c1170cbf33 100644 --- a/lib/classes/filetypes.php +++ b/lib/classes/filetypes.php @@ -292,6 +292,8 @@ abstract class core_filetypes { 'string' => 'audio'), 'webm' => array('type' => 'video/webm', 'icon' => 'video', 'groups' => array('html_video', 'video', 'web_video'), 'string' => 'video'), + 'webp' => ['type' => 'image/webp', 'icon' => 'image', 'groups' => ['image', 'web_image', 'optimised_image'], + 'string' => 'image'], 'wmv' => array('type' => 'video/x-ms-wmv', 'icon' => 'video', 'groups' => array('video'), 'string' => 'video'), 'asf' => array('type' => 'video/x-ms-asf', 'icon' => 'video', 'groups' => array('video'), 'string' => 'video'), 'wma' => array('type' => 'audio/x-ms-wma', 'icon' => 'audio', 'groups' => array('audio'), 'string' => 'audio'), diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index 7d861ec2637..673bf89cde1 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -1632,5 +1632,13 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2025031800.04); } + if ($oldversion < 2025032800.01) { + // Upgrade webp mime type for existing webp files. + upgrade_create_async_mimetype_upgrade_task('image/webp', ['webp']); + + // Main savepoint reached. + upgrade_main_savepoint(true, 2025032800.01); + } + return true; } diff --git a/version.php b/version.php index 7fb431ad347..95773d0d518 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2025032800.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2025032800.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes. $release = '5.0dev+ (Build: 20250328)'; // Human-friendly version name