From 0e442ee776f734dc43651cbd75ba930fdcd32f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Sat, 3 Oct 2015 11:14:35 +0200 Subject: [PATCH] MDL-49329 admin: Introduce new \core\update\core_manager tool The plan is to have a single tool looking after all operations with plugin ZIP packages (downloading, unzipping, moving to the dirroot and back). For legacy reasons, we have that logic currently duplicated in mdeploy and tool_installaddon. I would like to unify and simplify the whole machinery to use the same code for available updates, manual installation and plugin dependencies. --- lib/classes/plugin_manager.php | 112 ++++- lib/classes/update/code_manager.php | 396 ++++++++++++++++++ lib/tests/fixtures/bar.zip | Bin 0 -> 908 bytes lib/tests/fixtures/invalidroot.zip | Bin 0 -> 714 bytes .../fixtures/testable_update_code_manager.php | 53 +++ lib/tests/update_code_manager_test.php | 147 +++++++ 6 files changed, 706 insertions(+), 2 deletions(-) create mode 100644 lib/classes/update/code_manager.php create mode 100644 lib/tests/fixtures/bar.zip create mode 100644 lib/tests/fixtures/invalidroot.zip create mode 100644 lib/tests/fixtures/testable_update_code_manager.php create mode 100644 lib/tests/update_code_manager_test.php diff --git a/lib/classes/plugin_manager.php b/lib/classes/plugin_manager.php index b3083338ecf..aadcc6d9ecf 100644 --- a/lib/classes/plugin_manager.php +++ b/lib/classes/plugin_manager.php @@ -81,6 +81,8 @@ class core_plugin_manager { protected $presentplugins = null; /** @var array reordered list of plugin types */ protected $plugintypes = null; + /** @var \core\update\code_manager code manager to use for plugins code operations */ + protected $codemanager = null; /** * Direct initiation not allowed, use the factory method {@link self::instance()} @@ -933,14 +935,47 @@ class core_plugin_manager { } /** - * Return a list of all missing dependencies. + * Obtain the plugin ZIP file from the given URL + * + * The caller is supposed to know both downloads URL and the MD5 hash of + * the ZIP contents in advance, typically by using the API requests against + * the plugins directory. + * + * @param string $url + * @param string $md5 + * @return string|bool full path to the file, false on error + */ + public function get_remote_plugin_zip($url, $md5) { + return $this->get_code_manager()->get_remote_plugin_zip($url, $md5); + } + + /** + * Extracts the saved plugin ZIP file. + * + * Returns the list of files found in the ZIP. The format of that list is + * array of (string)filerelpath => (bool|string) where the array value is + * either true or a string describing the problematic file. + * + * @see zip_packer::extract_to_pathname() + * @param string $zipfilepath full path to the saved ZIP file + * @param string $targetdir full path to the directory to extract the ZIP file to + * @param string $rootdir explicitly rename the root directory of the ZIP into this non-empty value + * @param array list of extracted files as returned by {@link zip_packer::extract_to_pathname()} + */ + public function unzip_plugin_file($zipfilepath, $targetdir, $rootdir = '') { + return $this->get_code_manager()->unzip_plugin_file($zipfilepath, $targetdir, $rootdir); + } + + /** + * Return a list of missing dependencies. * * This should provide the full list of plugins that should be installed to * fulfill the requirements of all plugins, if possible. * + * @param bool $availableonly return only available missing dependencies * @return array of stdClass|bool indexed by the component name */ - public function missing_dependencies() { + public function missing_dependencies($availableonly=false) { $dependencies = array(); @@ -976,6 +1011,14 @@ class core_plugin_manager { } } + if ($availableonly) { + foreach ($dependencies as $component => $info) { + if (empty($info) or empty($info->version)) { + unset($dependencies[$component]); + } + } + } + return $dependencies; } @@ -1159,6 +1202,57 @@ class core_plugin_manager { return $this->is_directory_removable($pluginfo->rootdir); } + /** + * Is it possible to create a new plugin directory for the given plugin type? + * + * @throws coding_exception for invalid plugin types or non-existing plugin type locations + * @param string $plugintype + * @return boolean + */ + public function is_plugintype_writable($plugintype) { + + $plugintypepath = $this->get_plugintype_root($plugintype); + + if (is_null($plugintypepath)) { + throw new coding_exception('Unknown plugin type: '.$plugintype); + } + + if ($plugintypepath === false) { + throw new coding_exception('Plugin type location does not exist: '.$plugintype); + } + + return is_writable($plugintypepath); + } + + /** + * Returns the full path of the root of the given plugin type + * + * Null is returned if the plugin type is not known. False is returned if + * the plugin type root is expected but not found. Otherwise, string is + * returned. + * + * @param string $plugintype + * @return string|bool|null + */ + public function get_plugintype_root($plugintype) { + + $plugintypepath = null; + foreach (core_component::get_plugin_types() as $type => $fullpath) { + if ($type === $plugintype) { + $plugintypepath = $fullpath; + break; + } + } + if (is_null($plugintypepath)) { + return null; + } + if (!is_dir($plugintypepath)) { + return false; + } + + return $plugintypepath; + } + /** * Defines a list of all plugins that were originally shipped in the standard Moodle distribution, * but are not anymore and are deleted during upgrades. @@ -1570,4 +1664,18 @@ class core_plugin_manager { return true; } + + /** + * Returns a code_manager instance to be used for the plugins code operations. + * + * @return \core\update\code_manager + */ + protected function get_code_manager() { + + if ($this->codemanager === null) { + $this->codemanager = new \core\update\code_manager(); + } + + return $this->codemanager; + } } diff --git a/lib/classes/update/code_manager.php b/lib/classes/update/code_manager.php new file mode 100644 index 00000000000..a537f339289 --- /dev/null +++ b/lib/classes/update/code_manager.php @@ -0,0 +1,396 @@ +. + +/** + * Provides core\update\code_manager class. + * + * @package core_plugin + * @copyright 2012, 2013, 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\update; + +use coding_exception; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->libdir.'/filelib.php'); + +/** + * General purpose class managing the plugins source code files deployment + * + * The class is able and supposed to + * - fetch and cache ZIP files distributed via the Moodle Plugins directory + * - unpack the ZIP files in a temporary storage + * - archive existing version of the plugin source code + * - move (deploy) the plugin source code into the $CFG->dirroot + * + * @copyright 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class code_manager { + + /** @var string full path to the Moodle app directory root */ + protected $dirroot; + /** @var string full path to the temp directory root */ + protected $temproot; + + /** + * Instantiate the class instance + * + * @param string $dirroot full path to the moodle app directory root + * @param string $temproot full path to our temp directory + */ + public function __construct($dirroot=null, $temproot=null) { + global $CFG; + + if (empty($dirroot)) { + $dirroot = $CFG->dirroot; + } + + if (empty($temproot)) { + // Note we are using core_plugin here as that is the valid core + // subsystem we are part of. The namespace of this class (core\update) + // does not match it for legacy reasons. The data stored in the + // temp directory are expected to survive multiple requests and + // purging caches during the upgrade, so we make use of + // make_temp_directory(). The contents of it can be removed if needed, + // given the site is in the maintenance mode (so that cron is not + // executed) and the site is not being upgraded. + $temproot = make_temp_directory('core_plugin/code_manager'); + } + + $this->dirroot = $dirroot; + $this->temproot = $temproot; + + $this->init_temp_directories(); + } + + /** + * Obtain the plugin ZIP file from the given URL + * + * The caller is supposed to know both downloads URL and the MD5 hash of + * the ZIP contents in advance, typically by using the API requests against + * the plugins directory. + * + * @param string $url + * @param string $md5 + * @return string|bool full path to the file, false on error + */ + public function get_remote_plugin_zip($url, $md5) { + + // Sanitize and validate the URL. + $url = str_replace(array("\r", "\n"), '', $url); + + if (!preg_match('|^https?://|i', $url)) { + $this->debug('Error fetching plugin ZIP: unsupported transport protocol: '.$url); + return false; + } + + // The cache location for the file. + $distfile = $this->temproot.'/distfiles/'.$md5.'.zip'; + + if (is_readable($distfile)) { + return $distfile; + } + + // Download the file into a temporary location. + $tempdir = make_request_directory(); + $tempfile = $tempdir.'/plugin.zip'; + $result = $this->download_plugin_zip_file($url, $tempfile); + + if (!$result) { + return false; + } + + $actualmd5 = md5_file($tempfile); + + // Make sure the actual md5 hash matches the expected one. + if ($actualmd5 !== $md5) { + $this->debug('Error fetching plugin ZIP: md5 mismatch.'); + return false; + } + + // If the file is empty, something went wrong. + if ($actualmd5 === 'd41d8cd98f00b204e9800998ecf8427e') { + return false; + } + + // Store the file in our cache. + if (!rename($tempfile, $distfile)) { + return false; + } + + return $distfile; + } + + /** + * Move a folder with the plugin code from the source to the target location + * + * This can be used to move plugin folders to and from the dirroot/dataroot + * as needed. It is assumed that the caller checked that both locations are + * writable. + * + * Permissions in the target location are set to the same values that the + * parent directory has (see MDL-42110 for details). + * + * @param string $source full path to the current plugin code folder + * @param string $target full path to the new plugin code folder + */ + public function move_plugin_directory($source, $target) { + + $targetparent = dirname($target); + + if ($targetparent === '.') { + // No directory separator in $target.. + throw new coding_exception('Invalid target path', $target); + } + + if (!is_writable($targetparent)) { + throw new coding_exception('Attempting to move into non-writable parent directory', $targetparent); + } + + // Use parent directory's permissions for us, too. + $dirpermissions = fileperms($targetparent); + // Strip execute flags and use that for files. + $filepermissions = ($dirpermissions & 0666); + + $this->move_directory($source, $target, $dirpermissions, $filepermissions); + } + + /** + * Extracts the saved plugin ZIP file. + * + * Returns the list of files found in the ZIP. The format of that list is + * array of (string)filerelpath => (bool|string) where the array value is + * either true or a string describing the problematic file. + * + * @see zip_packer::extract_to_pathname() + * @param string $zipfilepath full path to the saved ZIP file + * @param string $targetdir full path to the directory to extract the ZIP file to + * @param string $rootdir explicitly rename the root directory of the ZIP into this non-empty value + * @param array list of extracted files as returned by {@link zip_packer::extract_to_pathname()} + */ + public function unzip_plugin_file($zipfilepath, $targetdir, $rootdir = '') { + + $fp = get_file_packer('application/zip'); + $files = $fp->extract_to_pathname($zipfilepath, $targetdir); + + if (!$files) { + return array(); + } + + if (!empty($rootdir)) { + $files = $this->rename_extracted_rootdir($targetdir, $rootdir, $files); + } + + // Sometimes zip may not contain all parent directories, add them to make it consistent. + foreach ($files as $path => $status) { + if ($status !== true) { + continue; + } + $parts = explode('/', trim($path, '/')); + while (array_pop($parts)) { + if (empty($parts)) { + break; + } + $dir = implode('/', $parts).'/'; + if (!isset($files[$dir])) { + $files[$dir] = true; + } + } + } + + return $files; + } + + // This is the end, my only friend, the end ... of external public API. + + /** + * Makes sure all temp directories exist and are writable. + */ + protected function init_temp_directories() { + make_writable_directory($this->temproot.'/distfiles'); + } + + /** + * Raise developer debugging level message. + * + * @param string $msg + */ + protected function debug($msg) { + debugging($msg, DEBUG_DEVELOPER); + } + + /** + * Download the ZIP file with the plugin package from the given location + * + * @param string $url URL to the file + * @param string $tofile full path to where to store the downloaded file + * @return bool false on error + */ + protected function download_plugin_zip_file($url, $tofile) { + + if (file_exists($tofile)) { + $this->debug('Error fetching plugin ZIP: target location exists.'); + return false; + } + + $status = $this->download_file_content($url, $tofile); + + if (!$status) { + $this->debug('Error fetching plugin ZIP.'); + @unlink($tofile); + return false; + } + + return true; + } + + /** + * Thin wrapper for the core's download_file_content() function. + * + * @param string $url URL to the file + * @param string $tofile full path to where to store the downloaded file + * @return bool + */ + protected function download_file_content($url, $tofile) { + + // Prepare the parameters for the download_file_content() function. + $headers = null; + $postdata = null; + $fullresponse = false; + $timeout = 300; + $connecttimeout = 20; + $skipcertverify = false; + $tofile = $tofile; + $calctimeout = false; + + return download_file_content($url, $headers, $postdata, $fullresponse, $timeout, + $connecttimeout, $skipcertverify, $tofile, $calctimeout); + } + + /** + * Internal helper method supposed to be called by self::move_plugin_directory() only. + * + * Moves the given source into a new location recursively. + * This is cross-device safe implementation to be used instead of the native rename() function. + * See https://bugs.php.net/bug.php?id=54097 for more details. + * + * @param string $source full path to the existing directory + * @param string $target full path to the new location of the directory + * @param int $dirpermissions + * @param int $filepermissions + */ + protected function move_directory($source, $target, $dirpermissions, $filepermissions) { + + if (file_exists($target)) { + throw new coding_exception('Attempting to overwrite existing directory', $target); + } + + if (is_dir($source)) { + $handle = opendir($source); + } else { + throw new coding_exception('Attempting to move non-existing source directory', $source); + } + + if (!file_exists($target)) { + // Do not use make_writable_directory() here - it is intended for dataroot only. + mkdir($target, true); + @chmod($target, $dirpermissions); + } + + if (!is_writable($target)) { + closedir($handle); + throw new coding_exception('Created folder not writable', $target); + } + + while ($filename = readdir($handle)) { + $sourcepath = $source.'/'.$filename; + $targetpath = $target.'/'.$filename; + + if ($filename === '.' or $filename === '..') { + continue; + } + + if (is_dir($sourcepath)) { + $this->move_directory($sourcepath, $targetpath, $dirpermissions, $filepermissions); + + } else { + rename($sourcepath, $targetpath); + @chmod($targetpath, $filepermissions); + } + } + + closedir($handle); + rmdir($source); + clearstatcache(); + } + + /** + * Renames the root directory of the extracted ZIP package. + * + * This method does not validate the presence of the single root directory + * (it is the validator's duty). It just searches for the first directory + * under the given location and renames it. + * + * The method will not rename the root if the requested location already + * exists. + * + * @param string $dirname fullpath location of the extracted ZIP package + * @param string $rootdir the requested name of the root directory + * @param array $files list of extracted files + * @return array eventually amended list of extracted files + */ + protected function rename_extracted_rootdir($dirname, $rootdir, array $files) { + + if (!is_dir($dirname)) { + $this->debug('Unable to rename rootdir of non-existing content'); + return $files; + } + + if (file_exists($dirname.'/'.$rootdir)) { + // This typically means the real root dir already has the $rootdir name. + return $files; + } + + $found = null; // The name of the first subdirectory under the $dirname. + foreach (scandir($dirname) as $item) { + if (substr($item, 0, 1) === '.') { + continue; + } + if (is_dir($dirname.'/'.$item)) { + $found = $item; + break; + } + } + + if (!is_null($found)) { + if (rename($dirname.'/'.$found, $dirname.'/'.$rootdir)) { + $newfiles = array(); + foreach ($files as $filepath => $status) { + $newpath = preg_replace('~^'.preg_quote($found.'/').'~', preg_quote($rootdir.'/'), $filepath); + $newfiles[$newpath] = $status; + } + return $newfiles; + } + } + + return $files; + } + +} diff --git a/lib/tests/fixtures/bar.zip b/lib/tests/fixtures/bar.zip new file mode 100644 index 0000000000000000000000000000000000000000..b190d2e6f7c8e6a7d7eb865b0b5a901d2652a28a GIT binary patch literal 908 zcmWIWW@h1H0D*%MJ+5E|lwe_yVMt0W(hm*cWMJmn*A=pHZ&yfZ1vdjD%L`@(1~3r- z*F7CgHz!ngPGVj<#E88>?Yr<9u>#Eq9%Lg@^B|_&2ATje38zCqrqnA;RnrD~1BAuV zOi9bnj|X~7uOOoU>@$#oAdF_9jXh8S7ne$LNl|8AdbD~$PHB2(US48us(P$~t%AB+ ze!c?G7)5n!E>Iwd0Bzy`2jTLMOw)aVyfh%@huU41T2!2wp9i-S6q+E6X6NRO`X@9V zJPOi0q3L`6%;l}x-nv@n&YTb45Ncp#U}9)uWMX8n!OcVc`E#)+&zC-%${@O8`P1je z)m6n-DxVZzDmE=mY^fOYQr(3NOXL{>ycwC~m~lm|1kiOLAi(g}5kx~GniUe!XwE`5 z2{W3JO|pZU1dVi{aaf}Y;V{f7M>ehz**I_l0Gfy;u8WC?IL*3+<|I+9W~JsqjJpjq3S=Z+k(v?@^0Zz-MgiF0Aj3fz&2SrgpaQM{Z$>6LW?aE20S!4Gn}-F@qP`+{swY1qU|JXe?oY>|~J9s6h=enh|K| dlEz24j7AT5To$o{qK|. + +/** + * @package core_plugin + * @category test + * @copyright 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\update; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Testable variant of \core\update\code_manager class. + * + * Provides access to some protected methods we want to explicitly test and + * bypass the actual cURL calls by providing fake responses. + * + * @copyright 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class testable_code_manager extends code_manager { + + /** @var int how many times $this->download_file_content() was called */ + public $downloadscounter = 0; + + /** + * Fake method to simulate fetching file via cURL. + * + * It simply creates a new file in the given location, the contents of + * which is the URL itself. + */ + protected function download_file_content($url, $tofile) { + $this->downloadscounter++; + file_put_contents($tofile, $url); + return true; + } +} diff --git a/lib/tests/update_code_manager_test.php b/lib/tests/update_code_manager_test.php new file mode 100644 index 00000000000..c811c2f785e --- /dev/null +++ b/lib/tests/update_code_manager_test.php @@ -0,0 +1,147 @@ +. + +/** + * Provides core_update_code_manager_testcase class. + * + * @package core + * @category test + * @copyright 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once(__DIR__.'/fixtures/testable_update_code_manager.php'); + +/** + * Tests for \core\update\code_manager features. + * + * @copyright 2015 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_update_code_manager_testcase extends advanced_testcase { + + public function test_get_remote_plugin_zip() { + $codeman = new \core\update\testable_code_manager(); + + $this->assertFalse($codeman->get_remote_plugin_zip('ftp://not.support.ed/', 'doesnotmatter')); + $this->assertDebuggingCalled('Error fetching plugin ZIP: unsupported transport protocol: ftp://not.support.ed/'); + + $this->assertEquals(0, $codeman->downloadscounter); + $this->assertFalse($codeman->get_remote_plugin_zip('http://first/', '')); + $this->assertDebuggingCalled('Error fetching plugin ZIP: md5 mismatch.'); + $this->assertEquals(1, $codeman->downloadscounter); + $this->assertNotFalse($codeman->get_remote_plugin_zip('http://first/', md5('http://first/'))); + $this->assertEquals(2, $codeman->downloadscounter); + $this->assertNotFalse($codeman->get_remote_plugin_zip('http://two/', md5('http://two/'))); + $this->assertEquals(3, $codeman->downloadscounter); + $this->assertNotFalse($codeman->get_remote_plugin_zip('http://first/', md5('http://first/'))); + $this->assertEquals(3, $codeman->downloadscounter); + } + + public function test_move_plugin_directory() { + $codeman = new \core\update\testable_code_manager(); + + $tmp = make_request_directory(); + $dir = make_writable_directory($tmp.'/mod/foo/lang/en'); + file_put_contents($dir.'/foo.txt', 'Hello world!'); + + $codeman->move_plugin_directory($tmp.'/mod/foo', $tmp.'/mod/.foo.2015100200'); + + $this->assertTrue(is_dir($tmp.'/mod')); + $this->assertFalse(is_dir($tmp.'/mod/foo')); + $this->assertTrue(is_file($tmp.'/mod/.foo.2015100200/lang/en/foo.txt')); + $this->assertSame('Hello world!', file_get_contents($tmp.'/mod/.foo.2015100200/lang/en/foo.txt')); + } + + /** + * @expectedException moodle_exception + */ + public function test_move_plugin_directory_invalid_target() { + $codeman = new \core\update\testable_code_manager(); + $codeman->move_plugin_directory(make_request_directory(), 'this_is_not_valid_path'); + } + + /** + * @expectedException moodle_exception + */ + public function test_move_plugin_directory_nonwritable_target() { + $codeman = new \core\update\testable_code_manager(); + // If this does not throw exception for you, please send me your IP address. + $codeman->move_plugin_directory(make_request_directory(), '/'); + } + + /** + * @expectedException moodle_exception + */ + public function test_move_plugin_directory_existing_target() { + $codeman = new \core\update\testable_code_manager(); + $dir1 = make_request_directory(); + $dir2 = make_request_directory(); + $codeman->move_plugin_directory($dir1, $dir2); + } + + /** + * @expectedException moodle_exception + */ + public function test_move_plugin_directory_nonexisting_source() { + $codeman = new \core\update\testable_code_manager(); + $codeman->move_plugin_directory(make_request_directory().'/source', make_request_directory().'/target'); + } + + public function test_unzip_plugin_file() { + $codeman = new \core\update\testable_code_manager(); + $zipfilepath = __DIR__.'/fixtures/invalidroot.zip'; + $targetdir = make_request_directory(); + + $files = $codeman->unzip_plugin_file($zipfilepath, $targetdir); + + $this->assertInternalType('array', $files); + $this->assertCount(4, $files); + $this->assertSame(true, $files['invalid-root/']); + $this->assertSame(true, $files['invalid-root/lang/']); + $this->assertSame(true, $files['invalid-root/lang/en/']); + $this->assertSame(true, $files['invalid-root/lang/en/fixed_root.php']); + foreach ($files as $file => $status) { + if (substr($file, -1) === '/') { + $this->assertTrue(is_dir($targetdir.'/'.$file)); + } else { + $this->assertTrue(is_file($targetdir.'/'.$file)); + } + } + + $files = $codeman->unzip_plugin_file($zipfilepath, $targetdir, 'fixed_root'); + + $this->assertInternalType('array', $files); + $this->assertCount(4, $files); + $this->assertSame(true, $files['fixed_root/']); + $this->assertSame(true, $files['fixed_root/lang/']); + $this->assertSame(true, $files['fixed_root/lang/en/']); + $this->assertSame(true, $files['fixed_root/lang/en/fixed_root.php']); + foreach ($files as $file => $status) { + if (substr($file, -1) === '/') { + $this->assertTrue(is_dir($targetdir.'/'.$file)); + } else { + $this->assertTrue(is_file($targetdir.'/'.$file)); + } + } + + $zipfilepath = __DIR__.'/fixtures/bar.zip'; + $files = $codeman->unzip_plugin_file($zipfilepath, $targetdir, 'bar'); + } +}