MDL-15919, MDL-15920 reworked support for archiving

This commit is contained in:
skodak 2008-08-08 10:22:59 +00:00
parent 3501d96b8b
commit 0b0bfa9345
9 changed files with 606 additions and 83 deletions

View File

@ -374,7 +374,9 @@ function unzip_file($zipfile, $destination = '', $showstatus_ignored = true) {
return false;
}
$result = get_file_packer()->unzip_files_to_pathname($zipfile, $destpath);
$packer = get_file_packer('application/zip');
$result = $packer->extract_to_pathname($zipfile, $destpath);
if ($result === false) {
return false;
@ -452,9 +454,9 @@ function zip_files ($originalfiles, $destination) {
$zipfiles[substr($file, $start)] = $file;
}
$packer = get_file_packer();
$packer = get_file_packer('application/zip');
return $packer->zip_files_to_pathname($zipfiles, $destfilename);
return $packer->archive_to_pathname($zipfiles, $destfilename);
}
/////////////////////////////////////////////////////////////

View File

@ -100,17 +100,18 @@ class stored_file {
/**
* Unzip file to given file path (real OS filesystem), existing files are overwrited
* @param string $path target directory
* @param object $file_packer
* @param string $pathname target directory
* @return mixed list of processed files; false if error
*/
public function unzip_files_to_pathname($path) {
$packer = get_file_packer();
$zipfile = $this->get_content_file_location();
return $packer->unzip_files_to_pathname($path, $path);
public function extract_to_pathname(file_packer $packer, $pathname) {
$archivefile = $this->get_content_file_location();
return $packer->extract_to_pathname($archivefile, $pathname);
}
/**
* Unzip file to given file path (real OS filesystem), existing files are overwrited
* @param object $file_packer
* @param int $contextid
* @param string $filearea
* @param int $itemid
@ -118,10 +119,9 @@ class stored_file {
* @param int $userid
* @return mixed list of processed files; false if error
*/
public function unzip_files_to_storage($contextid, $filearea, $itemid, $pathbase, $userid=null) {
$packer = get_file_packer();
$zipfile = $this->get_content_file_location();
return $packer->unzip_files_to_storage($zipfile, $contextid, $filearea, $itemid, $pathbase);
public function extract_to_storage(file_packer $packer, $contextid, $filearea, $itemid, $pathbase, $userid=null) {
$archivefile = $this->get_content_file_location();
return $packer->extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase);
}
/**
@ -130,15 +130,15 @@ class stored_file {
* @param string $archivepath pathname in zip archive
* @return bool success
*/
public function add_to_ziparchive(zip_archive $ziparch, $archivepath) {
public function archive_file(file_archive $filearch, $archivepath) {
if ($this->is_directory()) {
return $ziparch->addEmptyDir($archivepath);
return $filearch->add_directory($archivepath);
} else {
$path = $this->get_content_file_location();
if (!is_readable($path)) {
return false;
}
return $ziparch->addFile($path, $archivepath);
return $filearch->add_file_from_pathname($archivepath, $path);
}
}

View File

@ -1,11 +0,0 @@
<?php //$Id$
class zip_archive extends ZipArchive {
//TODO: limit number of open file handles by fetching small files into memory and closing/reopening archive for large files
//TODO: add file name encoding conversions
//TODO: prevent adding of target zip into archive
}

View File

@ -5,7 +5,8 @@ define('BYTESERVING_BOUNDARY', 's1k2o3d4a5k6s7'); //unique string constant
require_once("$CFG->libdir/file/file_exceptions.php");
require_once("$CFG->libdir/file/file_storage.php");
require_once("$CFG->libdir/file/file_browser.php");
require_once("$CFG->libdir/file/file_packer.php");
require_once("$CFG->libdir/packer/zip_packer.php");
function get_file_url($path, $options=null, $type='coursefile') {
global $CFG;

View File

@ -4535,22 +4535,33 @@ function get_file_browser() {
/**
* Returns file packer
* @param string $mimetype
* @return object file_storage
*/
function get_file_packer() {
function get_file_packer($mimetype='application/zip') {
global $CFG;
static $fp = null;
static $fp = array();;
if ($fp) {
return $fp;
if (isset($fp[$mimetype])) {
return $fp[$mimetype];
}
require_once("$CFG->libdir/filelib.php");
switch ($mimetype) {
case 'application/zip':
$classname = 'zip_packer';
break;
case 'application/x-tar':
// $classname = 'tar_packer';
// break;
default:
return false;
}
$fp = new file_packer();
require_once("$CFG->libdir/packer/$classname.php");
$fp[$mimetype] = new $classname();
return $fp;
return $fp[$mimetype];
}
/**

175
lib/packer/file_archive.php Normal file
View File

@ -0,0 +1,175 @@
<?php //$Id$
abstract class file_archive implements Iterator {
/** Open archive if exists, fail if does not exist. */
const OPEN = 1;
/** Open archive if exists, create if does not. */
const CREATE = 2;
/** Always create new archive */
const OVERWRITE = 4;
/** Encoding of file names - windows usually expects DOS single-byte charset*/
protected $encoding = 'utf-8';
/**
* Open or create archive (depending on $mode)
* @param string $archivepathname
* @param int $mode OPEN, CREATE or OVERWRITE constant
* @param string $encoding archive local paths encoding
* @return bool success
*/
public abstract function open($archivepathname, $mode=file_archive::CREATE, $encoding='utf-8');
/**
* Close archive
* @return bool success
*/
public abstract function close();
/**
* Returns file stream for reading of content
* @param int $index of file
* @return stream or false if error
*/
public abstract function get_stream($index);
/**
* Returns file information
* @param int $index of file
* @return info object or false if error
*/
public abstract function get_info($index);
/**
* Returns array of info about all files in archive
* @return array of file infos
*/
public abstract function list_files();
/**
* Returns number of files in archive
* @return int number of files
*/
public abstract function count();
/**
* Add file into archive
* @param string $localname name of file in archive
* @param string $pathname localtion of file
* @return bool success
*/
public abstract function add_file_from_pathname($localname, $pathname);
/**
* Add content of string into archive
* @param string $localname name of file in archive
* @param string $contents
* @return bool success
*/
public abstract function add_file_from_string($localname, $contents);
/**
* Add empty directory into archive
* @param string $local
* @return bool success
*/
public abstract function add_directory($localname);
/**
* Tries to convert $localname into another encoding,
* please note that it may fail really badly.
* @param strin $localname in utf-8 encoding
* @return string
*/
protected function mangle_pathname($localname) {
if ($this->encoding === 'utf-8') {
return $localname;
}
$textlib = textlib_get_instance();
$converted = $textlib->convert($localname, 'utf-8', $this->encoding);
$original = $textlib->convert($converted, $this->encoding, 'utf-8');
if ($original === $localname) {
$result = $converted;
} else {
// try ascci conversion
$converted2 = $textlib->specialtoascii($localname);
$converted2 = $textlib->convert($converted2, 'utf-8', $this->encoding);
$original2 = $textlib->convert($converted, $this->encoding, 'utf-8');
if ($original2 === $localname) {
//this looks much better
$result = $converted2;
} else {
//bad luck - the file name may not be usable at all
$result = $converted;
}
}
$result = ereg_replace('\.\.+', '', $result);
$result = ltrim($result); // no leadin /
if ($result === '.') {
$result = '';
}
return $result;
}
/**
* Tries to convert $localname into utf-8
* please note that it may fail really badly.
* The resulting file name is cleaned.
*
* @param strin $localname in anothe encoding
* @return string in utf-8
*/
protected function unmangle_pathname($localname) {
if ($this->encoding === 'utf-8') {
return $localname;
}
$textlib = textlib_get_instance();
$result = $textlib->convert($localname, $this->encoding, 'utf-8');
$result = clean_param($result, PARAM_PATH);
$result = ltrim($result); // no leadin /
return $result;
}
/**
* Returns current file info
* @return object
*/
//public abstract function current();
/**
* Returns the index of current file
* @return int current file index
*/
//public abstract function key();
/**
* Moves forward to next file
* @return void
*/
//public abstract function next();
/**
* Revinds back to the first file
* @return void
*/
//public abstract function rewind();
/**
* Did we reach the end?
* @return boolean
*/
//public abstract function valid();
}

View File

@ -0,0 +1,47 @@
<?php //$Id$
/**
* Abstract class for archiving of files.
*/
abstract class file_packer {
/**
* archive files and store the result in file storage
* @param array $archivepath=>$pathanme or stored file instance
* @param int $contextid
* @param string $filearea
* @param int $itemid
* @param string $filepath
* @param string $filename
* @return mixed false if error stored file instance if ok
*/
public abstract function archive_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null);
/**
* Archive files and store the result in os file
* @param array $archivepath=>$pathanme or stored file instance
* @param string $archivefile
* @return bool success
*/
public abstract function archive_to_pathname($files, $archivefile);
/**
* Extract file to given file path (real OS filesystem), existing files are overwrited
* @param mixed $archivefile full pathname of zip file or stored_file instance
* @param string $pathname target directory
* @return mixed list of processed files; false if error
*/
public abstract function extract_to_pathname($archivefile, $pathname);
/**
* Extract file to given file path (real OS filesystem), existing files are overwrited
* @param mixed $archivefile full pathname of zip file or stored_file instance
* @param int $contextid
* @param string $filearea
* @param int $itemid
* @param string $filepath
* @return mixed list of processed files; false if error
*/
public abstract function extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase, $userid=null);
}

304
lib/packer/zip_archive.php Normal file
View File

@ -0,0 +1,304 @@
<?php //$Id$
require_once("$CFG->libdir/packer/file_archive.php");
class zip_archive extends file_archive {
/** Pathname of archive */
protected $archivepathname = null;
/** Used memory tracking */
protected $usedmem = 0;
/** Iteration position */
protected $pos = 0;
/** TipArchive instance */
protected $za;
/**
* Open or create archive (depending on $mode)
* @param string $archivepathname
* @param int $mode OPEN, CREATE or OVERWRITE constant
* @param string $encoding archive local paths encoding
* @return bool success
*/
public function open($archivepathname, $mode=file_archive::CREATE, $encoding='utf-8') {
$this->close();
$this->usedmem = 0;
$this->pos = 0;
$this->za = new ZipArchive();
switch($mode) {
case file_archive::OPEN: $flags = 0; break;
case file_archive::OVERWRITE: $flags = ZIPARCHIVE::OVERWRITE; break;
case file_archive::CREATE:
default : $flags = ZIPARCHIVE::CREATE; break;
}
$result = $this->za->open($archivepathname, $flags);
if ($result === true) {
$this->encoding = $encoding;
if (file_exists($archivepathname)) {
$this->archivepathname = realpath($archivepathname);
} else {
$this->archivepathname = $archivepathname;
}
return true;
} else {
$this->za = null;
$this->archivepathname = null;
$this->encooding = 'utf-8';
// TODO: maybe we should return some error info
return false;
}
}
/**
* Close archive
* @return bool success
*/
public function close() {
if (!isset($this->za)) {
return false;
}
$res = $this->za->close();
$this->za = null;
return $res;
}
/**
* Returns file stream for reading of content
* @param int $index of file
* @return stream or false if error
*/
public function get_stream($index) {
if (!isset($this->za)) {
return false;
}
$name = $this->za->getNameIndex($index);
if ($name === false) {
return false;
}
return $this->za->getStream($name);
}
/**
* Returns file information
* @param int $index of file
* @return info object or false if error
*/
public function get_info($index) {
if (!isset($this->za)) {
return false;
}
if ($index < 0 or $index >=$this->count()) {
return false;
}
$result = $this->za->statIndex($index);
if ($result === false) {
return false;
}
$info = new object();
$info->index = $index;
$info->original_pathname = $result['name'];
$info->pathname = $this->unmangle_pathname($result['name']);
$info->mtime = (int)$result['mtime'];
if ($info->pathname[strlen($info->pathname)-1] === '/') {
$info->is_directory = true;
$info->size = 0;
} else {
$info->is_directory = false;
$info->size = (int)$result['size'];
}
return $info;
}
/**
* Returns array of info about all files in archive
* @return array of file infos
*/
public function list_files() {
if (!isset($this->za)) {
return false;
}
$infos = array();
for ($i=0; $i<$this->count(); $i++) {
$info = $this->get_info($i);
if ($info === false) {
continue;
}
$infos[$i] = $info;
}
return $infos;
}
/**
* Returns number of files in archive
* @return int number of files
*/
public function count() {
if (!isset($this->za)) {
return false;
}
return $this->za->numFiles;
}
/**
* Add file into archive
* @param string $localname name of file in archive
* @param string $pathname localtion of file
* @return bool success
*/
public function add_file_from_pathname($localname, $pathname) {
if (!isset($this->za)) {
return false;
}
if ($this->archivepathname === realpath($pathname)) {
// do not add self into archive
return false;
}
if (is_null($localname)) {
$localname = clean_param($pathname, PARAM_PATH);
}
$localname = trim($localname, '/'); // no leading slashes in archives
$localname = $this->mangle_pathname($localname);
if ($localname === '') {
//sorry - conversion failed badly
return false;
}
if ($this->count() > 0 and $this->count() % 500 === 0) {
// workaround for open file handles problem, ZipArchive uses file locking in order to prevent file modifications before the close() (strange, eh?)
$this->close();
$res = $this->open($this->archivepathname, file_archive::OPEN, $this->encoding);
if ($res !== true) {
error('Can not open zip file, probably zip extension bug on 64bit os'); //TODO ??
}
}
return $this->za->addFile($pathname, $localname);
}
/**
* Add content of string into archive
* @param string $localname name of file in archive
* @param string $contents
* @return bool success
*/
public function add_file_from_string($localname, $contents) {
if (!isset($this->za)) {
return false;
}
$localname = trim($localname, '/'); // no leading slashes in archives
$localname = $this->mangle_pathname($localname);
if ($localname === '') {
//sorry - conversion failed badly
return false;
}
if ($this->usedmem > 2097151) {
/// this prevents running out of memory when adding many large files using strings
$this->close();
$res = $this->open($this->archivepathname, file_archive::OPEN, $this->encoding);
if ($res !== true) {
error('Can not open zip file, probably zip extension bug on 64bit os'); //TODO ??
}
}
$this->usedmem += strlen($contents);
return $this->za->addFromString($localname, $contents);
}
/**
* Add empty directory into archive
* @param string $local
* @return bool success
*/
public function add_directory($localname) {
if (!isset($this->za)) {
return false;
}
$localname = ltrim($localname, '/'). '/';
$localname = $this->mangle_pathname($localname);
if ($localname === '/') {
//sorry - conversion failed badly
return false;
}
return $this->za->addEmptyDir($localname);
}
/**
* Returns current file info
* @return object
*/
public function current() {
if (!isset($this->za)) {
return false;
}
return $this->get_info($this->pos);
}
/**
* Returns the index of current file
* @return int current file index
*/
public function key() {
return $this->pos;
}
/**
* Moves forward to next file
* @return void
*/
public function next() {
$this->pos++;
}
/**
* Revinds back to the first file
* @return void
*/
public function rewind() {
$this->pos = 0;
}
/**
* Did we reach the end?
* @return boolean
*/
public function valid() {
if (!isset($this->za)) {
return false;
}
return ($this->pos < $this->count());
}
}

View File

@ -1,9 +1,12 @@
<?php //$Id$
require_once("$CFG->libdir/packer/file_packer.php");
require_once("$CFG->libdir/packer/zip_archive.php");
/**
* Utility class - handles all zipping and unzipping operations.
*/
class file_packer {
class zip_packer extends file_packer {
/**
* Zip files and store the result in file storage
@ -15,7 +18,7 @@ class file_packer {
* @param string $filename
* @return mixed false if error stored file instance if ok
*/
public function zip_files_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null) {
public function archive_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null) {
global $CFG;
$fs = get_file_storage();
@ -23,7 +26,7 @@ class file_packer {
check_dir_exists($CFG->dataroot.'/temp/zip', true, true);
$tmpfile = tempnam($CFG->dataroot.'/temp/zip', 'zipstor');
if ($result = $this->zip_files_to_pathname($files, $tmpfile)) {
if ($result = $this->archive_to_pathname($files, $tmpfile)) {
if ($file = $fs->get_file($contextid, $filearea, $itemid, $filepath, $filename)) {
if (!$file->delete()) {
@unlink($tmpfile);
@ -37,6 +40,7 @@ class file_packer {
$file_record->filepath = $filepath;
$file_record->filename = $filename;
$file_record->userid = $userid;
$result = $fs->create_file_from_pathname($file_record, $tmpfile);
}
@unlink($tmpfile);
@ -46,19 +50,18 @@ class file_packer {
/**
* Zip files and store the result in os file
* @param array $archivepath=>$pathanme or stored file instance
* @param string $zipfile
* @param string $archivefile
* @return bool success
*/
public function zip_files_to_pathname($files, $zipfile) {
public function archive_to_pathname($files, $archivefile) {
global $CFG;
require_once("$CFG->libdir/file/zip_archive.php");
if (!is_array($files)) {
return false;
}
$ziparch = new zip_archive();
if (!$ziparch->open($zipfile, ZIPARCHIVE::OVERWRITE)) {
if (!$ziparch->open($archivefile, file_archive::OVERWRITE)) {
return false;
}
@ -67,21 +70,21 @@ class file_packer {
if (is_null($file)) {
// empty directories have null as content
$ziparch->addEmptyDir($archivepath.'/');
$ziparch->add_directory($archivepath.'/');
} else if (is_string($file)) {
$this->add_os_file_to_zip($ziparch, $archivepath, $file);
$this->archive_pathname($ziparch, $archivepath, $file);
} else {
$this->add_stored_file_to_zip($ziparch, $archivepath, $file);
$this->archive_stored($ziparch, $archivepath, $file);
}
}
return $ziparch->close();
}
protected function add_stored_file_to_zip($ziparch, $archivepath, $file) {
$file->add_to_ziparchive($ziparch, $archivepath);
private function archive_stored($ziparch, $archivepath, $file) {
$file->archive_file($ziparch, $archivepath);
if (!$file->is_directory()) {
return;
@ -98,11 +101,11 @@ class file_packer {
if (!$file->is_directory()) {
$path = $path.$file->get_filename();
}
$file->add_to_ziparchive($ziparch, $path);
$file->archive_file($ziparch, $path);
}
}
protected function add_os_file_to_zip( $ziparch, $archivepath, $file) {
private function archive_pathname($ziparch, $archivepath, $file) {
if (!file_exists($file)) {
return;
}
@ -111,13 +114,12 @@ class file_packer {
if (!is_readable($file)) {
return;
}
$ziparch->addFile($file, $archivepath);
$ziparch->add_file_from_pathname($archivepath, $file);
return;
}
if (is_dir($file)) {
if ($archivepath !== '') {
$archivepath = $archivepath.'/';
$ziparch->addEmptyDir($archivepath);
$ziparch->add_directory($archivepath);
}
$files = new DirectoryIterator($file);
foreach ($files as $file) {
@ -125,7 +127,7 @@ class file_packer {
continue;
}
$newpath = $archivepath.$file->getFilename();
$this->add_os_file_to_zip($ziparch, $newpath, $file->getPathname());
$this->archive_pathname($ziparch, $newpath, $file->getPathname());
}
unset($files); //release file handles
return;
@ -134,42 +136,38 @@ class file_packer {
/**
* Unzip file to given file path (real OS filesystem), existing files are overwrited
* @param mixed $zipfile full pathname of zip file or stored_file instance
* @param mixed $archivefile full pathname of zip file or stored_file instance
* @param string $pathname target directory
* @return mixed list of processed files; false if error
*/
public function unzip_files_to_pathname($zipfile, $pathname) {
public function extract_to_pathname($archivefile, $pathname) {
global $CFG;
require_once("$CFG->libdir/file/zip_archive.php");
if (!is_string($zipfile)) {
return $zipfile->unzip_files_to_pathname($pathname);
if (!is_string($archivefile)) {
return $archivefile->extract_to_pathname($this, $pathname);
}
$processed = array();
$pathname = rtrim($pathname, '/');
if (!is_readable($zipfile)) {
if (!is_readable($archivefile)) {
return false;
}
$ziparch = new zip_archive();
if (!$ziparch->open($zipfile, ZIPARCHIVE::FL_NOCASE)) {
$ziparch = new zip_archive();
if (!$ziparch->open($archivefile, file_archive::OPEN)) {
return false;
}
for ($i=0; $i<$ziparch->numFiles; $i++) {
$index = $ziparch->statIndex($i);
$size = clean_param($index['size'], PARAM_INT);
$name = clean_param($index['name'], PARAM_PATH);
$name = ltrim($name, '/');
foreach ($ziparch as $info) {
$size = $info->size;
$name = $info->pathname;
if ($name === '' or array_key_exists($name, $processed)) {
//probably filename collisions caused by filename cleaning/conversion
continue;
}
if ($size === 0 and $name[strlen($name)-1] === '/') {
if ($info->is_directory) {
$newdir = "$pathname/$name";
// directory
if (is_file($newdir) and !unlink($newdir)) {
@ -205,7 +203,7 @@ class file_packer {
$processed[$name] = 'Can not write target file'; // TODO: localise
continue;
}
if (!$fz = $ziparch->getStream($index['name'])) {
if (!$fz = $ziparch->get_stream($info->index)) {
$processed[$name] = 'Can not read file from zip archive'; // TODO: localise
fclose($fp);
continue;
@ -231,18 +229,18 @@ class file_packer {
/**
* Unzip file to given file path (real OS filesystem), existing files are overwrited
* @param mixed $zipfile full pathname of zip file or stored_file instance
* @param mixed $archivefile full pathname of zip file or stored_file instance
* @param int $contextid
* @param string $filearea
* @param int $itemid
* @param string $filepath
* @return mixed list of processed files; false if error
*/
public function unzip_files_to_storage($zipfile, $contextid, $filearea, $itemid, $pathbase, $userid=null) {
public function extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase, $userid=null) {
global $CFG;
if (!is_string($zipfile)) {
return $zipfile->unzip_files_to_pathname($contextid, $filearea, $itemid, $pathbase, $userid);
if (!is_string($archivefile)) {
return $archivefile->extract_to_pathname($this, $contextid, $filearea, $itemid, $pathbase, $userid);
}
check_dir_exists($CFG->dataroot.'/temp/zip', true, true);
@ -254,24 +252,20 @@ class file_packer {
$processed = array();
$ziparch = new zip_archive();
if (!$ziparch->open($zipfile, ZIPARCHIVE::FL_NOCASE)) {
if (!$ziparch->open($archivefile, file_archive::OPEN)) {
return false;
}
for ($i=0; $i<$ziparch->numFiles; $i++) {
$index = $ziparch->statIndex($i);
$size = clean_param($index['size'], PARAM_INT);
$name = clean_param($index['name'], PARAM_PATH);
$name = ltrim($name, '/');
foreach ($ziparch as $info) {
$size = $info->size;
$name = $info->pathname;
if ($name === '' or array_key_exists($name, $processed)) {
//probably filename collisions caused by filename cleaning/conversion
continue;
}
if ($size === 0 and $name[strlen($name)-1] === '/') {
if ($info->is_directory) {
$newfilepath = $pathbase.$name.'/';
$fs->create_directory($contextid, $filearea, $itemid, $newfilepath, $userid);
$processed[$name] = true;
@ -287,7 +281,7 @@ class file_packer {
if ($size < 2097151) {
// small file
if (!$fz = $ziparch->getStream($index['name'])) {
if (!$fz = $ziparch->get_stream($info->index)) {
$processed[$name] = 'Can not read file from zip archive'; // TODO: localise
continue;
}
@ -332,7 +326,7 @@ class file_packer {
$processed[$name] = 'Can not write temp file'; // TODO: localise
continue;
}
if (!$fz = $ziparch->getStream($index['name'])) {
if (!$fz = $ziparch->get_stream($info->index)) {
@unlink($tmpfile);
$processed[$name] = 'Can not read file from zip archive'; // TODO: localise
continue;