mirror of
https://github.com/moodle/moodle.git
synced 2025-03-04 08:06:30 +01:00
c858655dde
A new packer for .tar.gz files (MIME type application/x-gzip) has been implemented, including unit tests. This packer is intended for use during backup/restore only and is not otherwise exposed in the user interface at present. However, it is supposed to follow the (ancient) POSIX .tar standard; files created with this packer will open in GNU tar. There are restrictions in support corresponding to the POSIX standard: - Non-ASCII filenames are not supported. - Very long file/path names are not supported. - Individual files within the .tar may not be more than 8GB. Some of these limitations might need to be addressed in future changes if the packer were to be used in areas other than backup/restore.
432 lines
17 KiB
PHP
432 lines
17 KiB
PHP
<?php
|
|
// This file is part of Moodle - http://moodle.org/
|
|
//
|
|
// Moodle is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// Moodle is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
/**
|
|
* Unit tests for /lib/filestorage/tgz_packer.php and tgz_extractor.php.
|
|
*
|
|
* @package core_files
|
|
* @copyright 2013 The Open University
|
|
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
|
|
*/
|
|
|
|
defined('MOODLE_INTERNAL') || die();
|
|
|
|
global $CFG;
|
|
require_once($CFG->libdir . '/filestorage/file_progress.php');
|
|
|
|
class core_files_tgz_packer_testcase extends advanced_testcase implements file_progress {
|
|
/**
|
|
* @var array Progress information passed to the progress reporter
|
|
*/
|
|
protected $progress;
|
|
|
|
/**
|
|
* Puts contents with specified time.
|
|
*
|
|
* @param string $path File path
|
|
* @param string $contents Contents of file
|
|
* @param int $mtime Time modified
|
|
*/
|
|
protected static function file_put_contents_at_time($path, $contents, $mtime) {
|
|
file_put_contents($path, $contents);
|
|
touch($path, $mtime);
|
|
}
|
|
|
|
/**
|
|
* Set up some files to be archived.
|
|
*
|
|
* @return array Array listing files of all types
|
|
*/
|
|
protected function prepare_file_list() {
|
|
global $CFG;
|
|
$this->resetAfterTest(true);
|
|
|
|
// Make array listing files to archive.
|
|
$filelist = array();
|
|
|
|
// Normal file.
|
|
self::file_put_contents_at_time($CFG->tempdir . '/file1.txt', 'File 1', 1377993601);
|
|
$filelist['out1.txt'] = $CFG->tempdir . '/file1.txt';
|
|
|
|
// Recursive directory w/ file and directory with file.
|
|
check_dir_exists($CFG->tempdir . '/dir1/dir2');
|
|
self::file_put_contents_at_time($CFG->tempdir . '/dir1/file2.txt', 'File 2', 1377993602);
|
|
self::file_put_contents_at_time($CFG->tempdir . '/dir1/dir2/file3.txt', 'File 3', 1377993603);
|
|
$filelist['out2'] = $CFG->tempdir . '/dir1';
|
|
|
|
// Moodle stored_file.
|
|
$context = context_system::instance();
|
|
$filerecord = array('contextid' => $context->id, 'component' => 'phpunit',
|
|
'filearea' => 'data', 'itemid' => 0, 'filepath' => '/',
|
|
'filename' => 'file4.txt', 'timemodified' => 1377993604);
|
|
$fs = get_file_storage();
|
|
$sf = $fs->create_file_from_string($filerecord, 'File 4');
|
|
$filelist['out3.txt'] = $sf;
|
|
|
|
// Moodle stored_file directory.
|
|
$filerecord['itemid'] = 1;
|
|
$filerecord['filepath'] = '/dir1/';
|
|
$filerecord['filename'] = 'file5.txt';
|
|
$filerecord['timemodified'] = 1377993605;
|
|
$fs->create_file_from_string($filerecord, 'File 5');
|
|
$filerecord['filepath'] = '/dir1/dir2/';
|
|
$filerecord['filename'] = 'file6.txt';
|
|
$filerecord['timemodified'] = 1377993606;
|
|
$fs->create_file_from_string($filerecord, 'File 6');
|
|
$filerecord['filepath'] = '/';
|
|
$filerecord['filename'] = 'excluded.txt';
|
|
$fs->create_file_from_string($filerecord, 'Excluded');
|
|
$filelist['out4'] = $fs->get_file($context->id, 'phpunit', 'data', 1, '/dir1/', '.');
|
|
|
|
// File stored as raw content.
|
|
$filelist['out5.txt'] = array('File 7');
|
|
|
|
// File where there's just an empty directory.
|
|
$filelist['out6'] = null;
|
|
|
|
return $filelist;
|
|
}
|
|
|
|
/**
|
|
* Tests getting the item.
|
|
*/
|
|
public function test_get_packer() {
|
|
$packer = get_file_packer('application/x-gzip');
|
|
$this->assertInstanceOf('tgz_packer', $packer);
|
|
}
|
|
|
|
/**
|
|
* Tests basic archive and extract to file paths.
|
|
*/
|
|
public function test_to_normal_files() {
|
|
global $CFG;
|
|
$packer = get_file_packer('application/x-gzip');
|
|
|
|
// Archive files.
|
|
$files = $this->prepare_file_list();
|
|
$archivefile = $CFG->tempdir . '/test.tar.gz';
|
|
$packer->archive_to_pathname($files, $archivefile);
|
|
|
|
// Extract same files.
|
|
$outdir = $CFG->tempdir . '/out';
|
|
check_dir_exists($outdir);
|
|
$result = $packer->extract_to_pathname($archivefile, $outdir);
|
|
|
|
// The result array should have file entries + directory entries for
|
|
// all implicit directories + entry for the explicit directory.
|
|
$expectedpaths = array('out1.txt', 'out2/', 'out2/dir2/', 'out2/dir2/file3.txt',
|
|
'out2/file2.txt', 'out3.txt', 'out4/', 'out4/dir2/', 'out4/file5.txt',
|
|
'out4/dir2/file6.txt', 'out5.txt', 'out6/');
|
|
sort($expectedpaths);
|
|
$actualpaths = array_keys($result);
|
|
sort($actualpaths);
|
|
$this->assertEquals($expectedpaths, $actualpaths);
|
|
foreach ($result as $path => $booleantrue) {
|
|
$this->assertTrue($booleantrue);
|
|
}
|
|
|
|
// Check the files are as expected.
|
|
$this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
|
|
$this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
|
|
$this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
|
|
$this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
|
|
$this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
|
|
$this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
|
|
$this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
|
|
$this->assertTrue(is_dir($outdir . '/out6'));
|
|
}
|
|
|
|
/**
|
|
* Tests archive and extract to Moodle file system.
|
|
*/
|
|
public function test_to_stored_files() {
|
|
global $CFG;
|
|
$packer = get_file_packer('application/x-gzip');
|
|
|
|
// Archive files.
|
|
$files = $this->prepare_file_list();
|
|
$archivefile = $CFG->tempdir . '/test.tar.gz';
|
|
$context = context_system::instance();
|
|
$sf = $packer->archive_to_storage($files,
|
|
$context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
|
|
$this->assertInstanceOf('stored_file', $sf);
|
|
|
|
// Extract (from storage) to disk.
|
|
$outdir = $CFG->tempdir . '/out';
|
|
check_dir_exists($outdir);
|
|
$packer->extract_to_pathname($sf, $outdir);
|
|
|
|
// Check the files are as expected.
|
|
$this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
|
|
$this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
|
|
$this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
|
|
$this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
|
|
$this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
|
|
$this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
|
|
$this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
|
|
$this->assertTrue(is_dir($outdir . '/out6'));
|
|
|
|
// Extract to Moodle storage.
|
|
$packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
|
|
$fs = get_file_storage();
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out1.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 1', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/', 'file2.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 2', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/dir2/', 'file3.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 3', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out3.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 4', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/', 'file5.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 5', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/dir2/', 'file6.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 6', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out5.txt');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertEquals('File 7', $out->get_content());
|
|
$out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out6/', '.');
|
|
$this->assertNotEmpty($out);
|
|
$this->assertTrue($out->is_directory());
|
|
|
|
// These functions are supposed to overwrite existing files; test they
|
|
// don't give errors when run twice.
|
|
$sf = $packer->archive_to_storage($files,
|
|
$context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
|
|
$this->assertInstanceOf('stored_file', $sf);
|
|
$packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
|
|
}
|
|
|
|
/**
|
|
* Tests extracting with a list of specified files.
|
|
*/
|
|
public function test_only_specified_files() {
|
|
global $CFG;
|
|
$packer = get_file_packer('application/x-gzip');
|
|
|
|
// Archive files.
|
|
$files = $this->prepare_file_list();
|
|
$archivefile = $CFG->tempdir . '/test.tar.gz';
|
|
$packer->archive_to_pathname($files, $archivefile);
|
|
|
|
// Extract same files.
|
|
$outdir = $CFG->tempdir . '/out';
|
|
check_dir_exists($outdir);
|
|
$result = $packer->extract_to_pathname($archivefile, $outdir,
|
|
array('out3.txt', 'out6/', 'out4/file5.txt'));
|
|
|
|
// Check result reporting only includes specified files.
|
|
$expectedpaths = array('out3.txt', 'out4/file5.txt', 'out6/');
|
|
sort($expectedpaths);
|
|
$actualpaths = array_keys($result);
|
|
sort($actualpaths);
|
|
$this->assertEquals($expectedpaths, $actualpaths);
|
|
|
|
// Check the files are as expected.
|
|
$this->assertFalse(file_exists($outdir . '/out1.txt'));
|
|
$this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
|
|
$this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
|
|
$this->assertTrue(is_dir($outdir . '/out6'));
|
|
}
|
|
|
|
/**
|
|
* Tests the progress reporting.
|
|
*/
|
|
public function test_file_progress() {
|
|
global $CFG;
|
|
|
|
// Set up.
|
|
$filelist = $this->prepare_file_list();
|
|
$packer = get_file_packer('application/x-gzip');
|
|
$archive = "$CFG->tempdir/archive.tgz";
|
|
$context = context_system::instance();
|
|
|
|
// Archive to pathname.
|
|
$this->progress = array();
|
|
$result = $packer->archive_to_pathname($filelist, $archive, true, $this);
|
|
$this->assertTrue($result);
|
|
// Should send progress at least once per file.
|
|
$this->assertTrue(count($this->progress) >= count($filelist));
|
|
// Progress should obey some restrictions.
|
|
$this->check_progress_toward_max();
|
|
|
|
// Archive to storage.
|
|
$this->progress = array();
|
|
$archivefile = $packer->archive_to_storage($filelist, $context->id,
|
|
'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
|
|
$this->assertInstanceOf('stored_file', $archivefile);
|
|
$this->assertTrue(count($this->progress) >= count($filelist));
|
|
$this->check_progress_toward_max();
|
|
|
|
// Extract to pathname.
|
|
$this->progress = array();
|
|
$target = "$CFG->tempdir/test/";
|
|
check_dir_exists($target);
|
|
$result = $packer->extract_to_pathname($archive, $target, null, $this);
|
|
remove_dir($target);
|
|
// We only output progress once per block, and this is kind of a small file.
|
|
$this->assertTrue(count($this->progress) >= 1);
|
|
$this->check_progress_toward_max();
|
|
|
|
// Extract to storage (from storage).
|
|
$this->progress = array();
|
|
$result = $packer->extract_to_storage($archivefile, $context->id,
|
|
'phpunit', 'target', 0, '/', null, $this);
|
|
$this->assertTrue(count($this->progress) >= 1);
|
|
$this->check_progress_toward_max();
|
|
|
|
// Extract to storage (from path).
|
|
$this->progress = array();
|
|
$result = $packer->extract_to_storage($archive, $context->id,
|
|
'phpunit', 'target', 0, '/', null, $this);
|
|
$this->assertTrue(count($this->progress) >= 1);
|
|
$this->check_progress_toward_max();
|
|
|
|
// Wipe created disk file.
|
|
unlink($archive);
|
|
}
|
|
|
|
/**
|
|
* Tests the list_files function with and without an index file.
|
|
*/
|
|
public function test_list_files() {
|
|
global $CFG;
|
|
|
|
// Set up.
|
|
$filelist = $this->prepare_file_list();
|
|
$packer = get_file_packer('application/x-gzip');
|
|
$archive = "$CFG->tempdir/archive.tgz";
|
|
|
|
// Archive with an index (default).
|
|
$packer = get_file_packer('application/x-gzip');
|
|
$result = $packer->archive_to_pathname($filelist, $archive, true, $this);
|
|
$this->assertTrue($result);
|
|
$hashwith = sha1_file($archive);
|
|
|
|
// List files.
|
|
$files = $packer->list_files($archive);
|
|
|
|
// Check they match expected.
|
|
$expectedinfo = array(
|
|
array('out1.txt', 1377993601, false, 6),
|
|
array('out2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
|
|
array('out2/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
|
|
array('out2/dir2/file3.txt', 1377993603, false, 6),
|
|
array('out2/file2.txt', 1377993602, false, 6),
|
|
array('out3.txt', 1377993604, false, 6),
|
|
array('out4/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
|
|
array('out4/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
|
|
array('out4/dir2/file6.txt', 1377993606, false, 6),
|
|
array('out4/file5.txt', 1377993605, false, 6),
|
|
array('out5.txt', tgz_packer::DEFAULT_TIMESTAMP, false, 6),
|
|
array('out6/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
|
|
);
|
|
$this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
|
|
|
|
// Archive with no index. Should have same result.
|
|
$this->progress = array();
|
|
$packer->set_include_index(false);
|
|
$result = $packer->archive_to_pathname($filelist, $archive, true, $this);
|
|
$this->assertTrue($result);
|
|
$hashwithout = sha1_file($archive);
|
|
$files = $packer->list_files($archive);
|
|
$this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
|
|
|
|
// Check it actually is different (does have index in)!
|
|
$this->assertNotEquals($hashwith, $hashwithout);
|
|
|
|
// Put the index back on in case of future tests.
|
|
$packer->set_include_index(true);
|
|
}
|
|
|
|
/**
|
|
* Utility function to convert the file info array into a simpler format
|
|
* for making comparisons.
|
|
*
|
|
* @param array $files Array from list_files result
|
|
*/
|
|
protected static function convert_info_for_assert(array $files) {
|
|
$actualinfo = array();
|
|
foreach ($files as $file) {
|
|
$actualinfo[] = array($file->pathname, $file->mtime, $file->is_directory, $file->size);
|
|
}
|
|
usort($actualinfo, function($a, $b) {
|
|
return strcmp($a[0], $b[0]);
|
|
});
|
|
return $actualinfo;
|
|
}
|
|
|
|
public function test_is_tgz_file() {
|
|
global $CFG;
|
|
|
|
// Set up.
|
|
$filelist = $this->prepare_file_list();
|
|
$packer1 = get_file_packer('application/x-gzip');
|
|
$packer2 = get_file_packer('application/zip');
|
|
$archive2 = "$CFG->tempdir/archive.zip";
|
|
|
|
// Archive in tgz and zip format.
|
|
$context = context_system::instance();
|
|
$archive1 = $packer1->archive_to_storage($filelist, $context->id,
|
|
'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
|
|
$this->assertInstanceOf('stored_file', $archive1);
|
|
$result = $packer2->archive_to_pathname($filelist, $archive2);
|
|
$this->assertTrue($result);
|
|
|
|
// Use is_tgz_file to detect which is which. First check is from storage,
|
|
// second check is from filesystem.
|
|
$this->assertTrue(tgz_packer::is_tgz_file($archive1));
|
|
$this->assertFalse(tgz_packer::is_tgz_file($archive2));
|
|
}
|
|
|
|
/**
|
|
* Checks that progress reported is numeric rather than indeterminate,
|
|
* and follows the progress reporting rules.
|
|
*/
|
|
protected function check_progress_toward_max() {
|
|
$lastvalue = -1; $lastmax = -1;
|
|
foreach ($this->progress as $progressitem) {
|
|
list($value, $max) = $progressitem;
|
|
if ($lastmax != -1) {
|
|
$this->assertEquals($max, $lastmax);
|
|
} else {
|
|
$lastmax = $max;
|
|
}
|
|
$this->assertTrue(is_integer($value));
|
|
$this->assertTrue(is_integer($max));
|
|
$this->assertNotEquals(file_progress::INDETERMINATE, $max);
|
|
$this->assertTrue($value <= $max);
|
|
$this->assertTrue($value >= $lastvalue);
|
|
$lastvalue = $value;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Handles file_progress interface.
|
|
*
|
|
* @param int $progress
|
|
* @param int $max
|
|
*/
|
|
public function progress($progress = file_progress::INDETERMINATE, $max = file_progress::INDETERMINATE) {
|
|
$this->progress[] = array($progress, $max);
|
|
}
|
|
}
|