diff --git a/admin/settings/development.php b/admin/settings/development.php index 44d25462017..6be22df803b 100644 --- a/admin/settings/development.php +++ b/admin/settings/development.php @@ -19,6 +19,15 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page $enablecssoptimiser->set_updatedcallback('theme_reset_all_caches'); $temp->add($enablecssoptimiser); + // Backup archive .mbz format: switching to .tar.gz enables larger files, better + // progress reporting and possibly better performance. This is an experimental + // setting but if successful, should be removed and enabled by default in a future + // version. Note: this setting controls newly-created backups only; restore always + // supports both formats. + $temp->add(new admin_setting_configcheckbox('enabletgzbackups', + new lang_string('enabletgzbackups', 'admin'), + new lang_string('enabletgzbackups_desc', 'admin'), 0)); + $ADMIN->add('experimental', $temp); // "debugging" settingpage diff --git a/backup/moodle2/backup_stepslib.php b/backup/moodle2/backup_stepslib.php index 922318c349b..0a7faef802b 100644 --- a/backup/moodle2/backup_stepslib.php +++ b/backup/moodle2/backup_stepslib.php @@ -1704,7 +1704,7 @@ class backup_zip_contents extends backup_execution_step implements file_progress $zipfile = $basepath . '/backup.mbz'; // Get the zip packer - $zippacker = get_file_packer('application/zip'); + $zippacker = get_file_packer('application/vnd.moodle.backup'); // Zip files $zippacker->archive_to_pathname($files, $zipfile, true, $this); diff --git a/backup/util/helper/backup_general_helper.class.php b/backup/util/helper/backup_general_helper.class.php index 03225a38aae..9e9d2c289f1 100644 --- a/backup/util/helper/backup_general_helper.class.php +++ b/backup/util/helper/backup_general_helper.class.php @@ -243,7 +243,8 @@ abstract class backup_general_helper extends backup_helper { // Extract moodle_backup.xml. $tmpname = 'info_from_mbz_' . time() . '_' . random_string(4); $tmpdir = $CFG->tempdir . '/backup/' . $tmpname; - $fp = get_file_packer('application/vnd.moodle.backup'); + $packer = get_file_packer('application/vnd.moodle.backup'); + $extracted = $fp->extract_to_pathname($filepath, $tmpdir, array('moodle_backup.xml')); $moodlefile = $tmpdir . '/' . 'moodle_backup.xml'; if (!$extracted || !is_readable($moodlefile)) { diff --git a/backup/util/ui/restore_ui_stage.class.php b/backup/util/ui/restore_ui_stage.class.php index 327722e72e3..dd58ca876ce 100644 --- a/backup/util/ui/restore_ui_stage.class.php +++ b/backup/util/ui/restore_ui_stage.class.php @@ -212,7 +212,7 @@ class restore_ui_stage_confirm extends restore_ui_independent_stage implements f $this->filepath = restore_controller::get_tempdir_name($this->contextid, $USER->id); - $fb = get_file_packer(); + $fb = get_file_packer('application/vnd.moodle.backup'); $result = $fb->extract_to_pathname("$CFG->tempdir/backup/".$this->filename, "$CFG->tempdir/backup/$this->filepath/", null, $this); diff --git a/lang/en/admin.php b/lang/en/admin.php index 71a73d89c36..2a2205c3039 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -487,6 +487,8 @@ $string['enablerecordcache'] = 'Enable record cache'; $string['enablerssfeeds'] = 'Enable RSS feeds'; $string['enablesafebrowserintegration'] = 'Enable Safe Exam Browser integration'; $string['enablestats'] = 'Enable statistics'; +$string['enabletgzbackups'] = 'Enable new backup format'; +$string['enabletgzbackups_desc'] = 'If enabled, future backups will be created in a new compression format for .mbz files (internally stored as a .tar.gz file). This removes the 4GB backup size restriction and may improve performance. Restore supports both formats and the difference should be transparent to users.'; $string['enabletrusttext'] = 'Enable trusted content'; $string['enablewebservices'] = 'Enable web services'; $string['enablewsdocumentation'] = 'Web services documentation'; diff --git a/lib/filestorage/mbz_packer.php b/lib/filestorage/mbz_packer.php new file mode 100644 index 00000000000..e149e35b284 --- /dev/null +++ b/lib/filestorage/mbz_packer.php @@ -0,0 +1,170 @@ +. + +/** + * Implementation of .mbz packer. + * + * This packer supports .mbz files which can be either .zip or .tar.gz format + * internally. A suitable format is chosen depending on system option when + * creating new files. + * + * Internally this packer works by wrapping the existing .zip/.tar.gz packers. + * + * Backup filenames do not contain non-ASCII characters so packers that do not + * support UTF-8 (like the current .tar.gz packer, and possibly external zip + * software in some cases if used) can be used by this packer. + * + * @package core_files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once("$CFG->libdir/filestorage/file_packer.php"); + +/** + * Utility class - handles all packing/unpacking of .mbz files. + * + * @package core_files + * @category files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class mbz_packer extends file_packer { + /** + * Archive files and store the result in file storage. + * + * Any existing file at that location will be overwritten. + * + * @param array $files array from archive path => pathname or stored_file + * @param int $contextid context ID + * @param string $component component + * @param string $filearea file area + * @param int $itemid item ID + * @param string $filepath file path + * @param string $filename file name + * @param int $userid user ID + * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error + * @param file_progress $progress Progress indicator callback or null if not required + * @return stored_file|bool false if error stored_file instance if ok + * @throws file_exception If file operations fail + * @throws coding_exception If any archive paths do not meet the restrictions + */ + public function archive_to_storage(array $files, $contextid, + $component, $filearea, $itemid, $filepath, $filename, + $userid = null, $ignoreinvalidfiles = true, file_progress $progress = null) { + return $this->get_packer_for_archive_operation()->archive_to_storage($files, + $contextid, $component, $filearea, $itemid, $filepath, $filename, + $userid, $ignoreinvalidfiles, $progress); + } + + /** + * Archive files and store the result in an OS file. + * + * @param array $files array from archive path => pathname or stored_file + * @param string $archivefile path to target zip file + * @param bool $ignoreinvalidfiles true means ignore missing or invalid files, false means abort on any error + * @param file_progress $progress Progress indicator callback or null if not required + * @return bool true if file created, false if not + * @throws coding_exception If any archive paths do not meet the restrictions + */ + public function archive_to_pathname(array $files, $archivefile, + $ignoreinvalidfiles=true, file_progress $progress = null) { + return $this->get_packer_for_archive_operation()->archive_to_pathname($files, + $archivefile, $ignoreinvalidfiles, $progress); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param stored_file|string $archivefile full pathname of zip file or stored_file instance + * @param string $pathname target directory + * @param array $onlyfiles only extract files present in the array + * @param file_progress $progress Progress indicator callback or null if not required + * @return array list of processed files (name=>true) + * @throws moodle_exception If error + */ + public function extract_to_pathname($archivefile, $pathname, + array $onlyfiles = null, file_progress $progress = null) { + return $this->get_packer_for_read_operation($archivefile)->extract_to_pathname( + $archivefile, $pathname, $onlyfiles, $progress); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param string|stored_file $archivefile full pathname of zip file or stored_file instance + * @param int $contextid context ID + * @param string $component component + * @param string $filearea file area + * @param int $itemid item ID + * @param string $pathbase file path + * @param int $userid user ID + * @param file_progress $progress Progress indicator callback or null if not required + * @return array list of processed files (name=>true) + * @throws moodle_exception If error + */ + public function extract_to_storage($archivefile, $contextid, + $component, $filearea, $itemid, $pathbase, $userid = null, + file_progress $progress = null) { + return $this->get_packer_for_read_operation($archivefile)->extract_to_storage( + $archivefile, $contextid, $component, $filearea, $itemid, $pathbase, + $userid, $progress); + } + + /** + * Returns array of info about all files in archive. + * + * @param string|stored_file $archivefile + * @return array of file infos + */ + public function list_files($archivefile) { + return $this->get_packer_for_read_operation($archivefile)->list_files($archivefile); + } + + /** + * Selects appropriate packer for new archive depending on system option. + * + * @return file_packer Suitable packer + */ + protected function get_packer_for_archive_operation() { + global $CFG; + + if ($CFG->enabletgzbackups) { + return get_file_packer('application/x-gzip'); + } else { + return get_file_packer('application/zip'); + } + } + + /** + * Selects appropriate packer for existing archive depending on file contents. + * + * @param string|stored_file $archivefile full pathname of zip file or stored_file instance + * @return file_packer Suitable packer + */ + protected function get_packer_for_read_operation($archivefile) { + global $CFG; + require_once($CFG->dirroot . '/lib/filestorage/tgz_packer.php'); + + if (tgz_packer::is_tgz_file($archivefile)) { + return get_file_packer('application/x-gzip'); + } else { + return get_file_packer('application/zip'); + } + } +} diff --git a/lib/filestorage/tests/mbz_packer_test.php b/lib/filestorage/tests/mbz_packer_test.php new file mode 100644 index 00000000000..cdb7bef92fb --- /dev/null +++ b/lib/filestorage/tests/mbz_packer_test.php @@ -0,0 +1,91 @@ +. + +/** + * Unit tests for /lib/filestorage/mbz_packer.php. + * + * @package core_files + * @copyright 2013 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->libdir . '/filestorage/file_progress.php'); + +class core_files_mbz_packer_testcase extends advanced_testcase { + + public function test_archive_with_both_options() { + global $CFG; + $this->resetAfterTest(); + + // Get backup packer. + $packer = get_file_packer('application/vnd.moodle.backup'); + + // Set up basic archive contents. + $files = array('1.txt' => array('frog')); + + // Create 2 archives (each with one file in) in default mode. + $CFG->enabletgzbackups = false; + $filefalse = $CFG->tempdir . '/false.mbz'; + $this->assertNotEmpty($packer->archive_to_pathname($files, $filefalse)); + $context = context_system::instance(); + $this->assertNotEmpty($storagefalse = $packer->archive_to_storage( + $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz')); + + // Create 2 archives in tgz mode. + $CFG->enabletgzbackups = true; + $filetrue = $CFG->tempdir . '/true.mbz'; + $this->assertNotEmpty($packer->archive_to_pathname($files, $filetrue)); + $context = context_system::instance(); + $this->assertNotEmpty($storagetrue = $packer->archive_to_storage( + $files, $context->id, 'phpunit', 'data', 0, '/', 'false.mbz')); + + // Check the sizes are different (indicating different formats). + $this->assertNotEquals(filesize($filefalse), filesize($filetrue)); + $this->assertNotEquals($storagefalse->get_filesize(), $storagetrue->get_filesize()); + + // Extract files into storage and into filesystem from both formats. + // (Note: the setting does not matter, but set to false just to check.) + $CFG->enabletgzbackups = false; + + // Extract to path (zip). + $packer->extract_to_pathname($filefalse, $CFG->tempdir); + $onefile = $CFG->tempdir . '/1.txt'; + $this->assertEquals('frog', file_get_contents($onefile)); + unlink($onefile); + + // Extract to path (tgz). + $packer->extract_to_pathname($filetrue, $CFG->tempdir); + $onefile = $CFG->tempdir . '/1.txt'; + $this->assertEquals('frog', file_get_contents($onefile)); + unlink($onefile); + + // Extract to storage (zip). + $packer->extract_to_storage($storagefalse, $context->id, 'phpunit', 'data', 1, '/'); + $fs = get_file_storage(); + $out = $fs->get_file($context->id, 'phpunit', 'data', 1, '/', '1.txt'); + $this->assertNotEmpty($out); + $this->assertEquals('frog', $out->get_content()); + + // Extract to storage (tgz). + $packer->extract_to_storage($storagetrue, $context->id, 'phpunit', 'data', 2, '/'); + $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/', '1.txt'); + $this->assertNotEmpty($out); + $this->assertEquals('frog', $out->get_content()); + } +} diff --git a/lib/moodlelib.php b/lib/moodlelib.php index d4c47b81e23..6df44149b17 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -6149,14 +6149,18 @@ function get_file_packer($mimetype='application/zip') { switch ($mimetype) { case 'application/zip': - case 'application/vnd.moodle.backup': case 'application/vnd.moodle.profiling': $classname = 'zip_packer'; break; + case 'application/x-gzip' : $classname = 'tgz_packer'; break; + case 'application/vnd.moodle.backup': + $classname = 'mbz_packer'; + break; + default: return false; }