From 3b4240bbae07814625cfcf6cc00cb50925cfaa58 Mon Sep 17 00:00:00 2001 From: Nick Liu Date: Fri, 27 Dec 2019 11:33:21 -0600 Subject: [PATCH] Quality control for e_file::unzipGithubArchive() - MOD: PHPDoc for e_file::unzipGithubArchive() - NEW: e_file::unzipGithubArchive(): Added exclusions for files that don't exist in production - NEW: e_file::unzipGithubArchive(): Accept a destination path argument for a custom extraction location - NEW: Restored unimplemented skipped list in e_file::unzipGithubArchive() - FIX: e_file::unzipGithubArchive(): Extraction fails if parent directory of file doesn't exist - MOD: Type hint for Base::$deployer - NEW: Basic test for e_file::unzipGithubArchive() --- e107_handlers/file_class.php | 76 +-- e107_tests/tests/_support/Helper/Base.php | 3 + e107_tests/tests/unit/e_fileTest.php | 655 ++++++++++++---------- 3 files changed, 414 insertions(+), 320 deletions(-) diff --git a/e107_handlers/file_class.php b/e107_handlers/file_class.php index ee334ec9c..f05e0ebd6 100644 --- a/e107_handlers/file_class.php +++ b/e107_handlers/file_class.php @@ -1664,10 +1664,16 @@ class e_file } - - - - public function unzipGithubArchive($url='core') + /** + * Download and extract a zipped copy of e107 + * @param string $url "core" to download the e107 core from Git master or + * a custom download URL + * @param string $destination_path The e107 root where the downloaded archive should be extracted, + * with a directory separator at the end + * @return array|bool FALSE on failure; + * An array of successful and failed path extractions + */ + public function unzipGithubArchive($url='core', $destination_path = e_BASE) { switch($url) @@ -1675,8 +1681,21 @@ class e_file case "core": $localfile = 'e107-master.zip'; $remotefile = 'https://codeload.github.com/e107inc/e107/zip/master'; - $excludes = array('e107-master/install.php','e107-master/favicon.ico'); - $excludeMatch = false; + $excludes = array( + 'e107-master/.codeclimate.yml', + 'e107-master/.editorconfig', + 'e107-master/.gitignore', + 'e107-master/.gitmodules', + 'e107-master/CONTRIBUTING.md', # moved to ./.github/CONTRIBUTING.md + 'e107-master/LICENSE', + 'e107-master/README.md', + 'e107-master/install.php', + 'e107-master/favicon.ico', + ); + $excludeMatch = array( + '/.github/', + '/e107_tests/', + ); break; // language. @@ -1712,18 +1731,18 @@ class e_file $excludes[] = $zipBase; $newFolders = array( - $zipBase.'/e107_admin/' => e_BASE.e107::getFolder('ADMIN'), - $zipBase.'/e107_core/' => e_BASE.e107::getFolder('CORE'), - $zipBase.'/e107_docs/' => e_BASE.e107::getFolder('DOCS'), - $zipBase.'/e107_handlers/' => e_BASE.e107::getFolder('HANDLERS'), - $zipBase.'/e107_images/' => e_BASE.e107::getFolder('IMAGES'), - $zipBase.'/e107_languages/' => e_BASE.e107::getFolder('LANGUAGES'), - $zipBase.'/e107_media/' => e_BASE.e107::getFolder('MEDIA'), - $zipBase.'/e107_plugins/' => e_BASE.e107::getFolder('PLUGINS'), - $zipBase.'/e107_system/' => e_BASE.e107::getFolder('SYSTEM'), - $zipBase.'/e107_themes/' => e_BASE.e107::getFolder('THEMES'), - $zipBase.'/e107_web/' => e_BASE.e107::getFolder('WEB'), - $zipBase.'/' => e_BASE + $zipBase.'/e107_admin/' => $destination_path.e107::getFolder('ADMIN'), + $zipBase.'/e107_core/' => $destination_path.e107::getFolder('CORE'), + $zipBase.'/e107_docs/' => $destination_path.e107::getFolder('DOCS'), + $zipBase.'/e107_handlers/' => $destination_path.e107::getFolder('HANDLERS'), + $zipBase.'/e107_images/' => $destination_path.e107::getFolder('IMAGES'), + $zipBase.'/e107_languages/' => $destination_path.e107::getFolder('LANGUAGES'), + $zipBase.'/e107_media/' => $destination_path.e107::getFolder('MEDIA'), + $zipBase.'/e107_plugins/' => $destination_path.e107::getFolder('PLUGINS'), + $zipBase.'/e107_system/' => $destination_path.e107::getFolder('SYSTEM'), + $zipBase.'/e107_themes/' => $destination_path.e107::getFolder('THEMES'), + $zipBase.'/e107_web/' => $destination_path.e107::getFolder('WEB'), + $zipBase.'/' => $destination_path ); $srch = array_keys($newFolders); @@ -1734,35 +1753,28 @@ class e_file $error = array(); $success = array(); - // $skipped = array(); + $skipped = array(); foreach($unarc as $k=>$v) { - if($this->matchFound($v['stored_filename'],$excludeMatch)) - { - continue; - } - - if(in_array($v['stored_filename'],$excludes)) + if($this->matchFound($v['stored_filename'],$excludeMatch) || + in_array($v['stored_filename'],$excludes)) { + $skipped[] = $v['stored_filename']; continue; } $oldPath = $v['filename']; $newPath = str_replace($srch,$repl, $v['stored_filename']); -/* - $success[] = $newPath; - continue;*/ - if($v['folder'] ==1 && is_dir($newPath)) { // $skipped[] = $newPath. " (already exists)"; continue; } - + mkdir(dirname($newPath), 0755, true); if(!rename($oldPath,$newPath)) { $error[] = $newPath; @@ -1774,9 +1786,7 @@ class e_file } - - return array('success'=>$success, 'error'=>$error); - + return array('success'=>$success, 'error'=>$error, 'skipped'=>$skipped); } diff --git a/e107_tests/tests/_support/Helper/Base.php b/e107_tests/tests/_support/Helper/Base.php index 53aaf5369..8a29a54a7 100644 --- a/e107_tests/tests/_support/Helper/Base.php +++ b/e107_tests/tests/_support/Helper/Base.php @@ -7,6 +7,9 @@ include_once(codecept_root_dir() . "lib/deployers/DeployerFactory.php"); abstract class Base extends \Codeception\Module { + /** + * @var \Deployer + */ protected $deployer; protected $deployer_components = ['db', 'fs']; diff --git a/e107_tests/tests/unit/e_fileTest.php b/e107_tests/tests/unit/e_fileTest.php index 15d3cd2ea..a8eadee65 100644 --- a/e107_tests/tests/unit/e_fileTest.php +++ b/e107_tests/tests/unit/e_fileTest.php @@ -1,303 +1,384 @@ fl = $this->make('e_file'); - } - catch (Exception $e) - { - $this->fail("Couldn't load e_file object"); - } + $this->fl = $this->make('e_file'); + } + catch (Exception $e) + { + $this->fail("Couldn't load e_file object"); + } - $this->exploitFile = e_TEMP."test_exploit_file.jpg"; + $this->exploitFile = e_TEMP."test_exploit_file.jpg"; - $content = ""; + $content = ""; - file_put_contents($this->exploitFile,$content); + file_put_contents($this->exploitFile,$content); - $this->filetypesFile = e_SYSTEM."filetypes.xml"; + $this->filetypesFile = e_SYSTEM."filetypes.xml"; - $content = ' + $content = ' '; - file_put_contents($this->filetypesFile, $content); + file_put_contents($this->filetypesFile, $content); - } - - protected function _after() - { - unlink($this->exploitFile); - unlink($this->filetypesFile); - } - - - public function testIsClean() - { - - $isCleanTest = array( - array('path'=>$this->exploitFile, 'expected' => false), // suspicious - array('path'=>e_SYSTEM."filetypes.xml", 'expected' => true), // okay - array('path'=>e_PLUGIN."gallery/images/butterfly.jpg", 'expected' => true), // okay - ); - - foreach($isCleanTest as $file) - { - $actual = $this->fl->isClean($file['path'], $file['path']); - $this->assertEquals($file['expected'],$actual, "isClean() failed on {$file['path']} with error code: ".$this->fl->getErrorCode()); - } - - } - - public function testGetAllowedFileTypes() - { - $actual = $this->fl->getAllowedFileTypes(); - - $expected = array ( - 'zip' => 2097152, // 2M in bytes - 'gz' => 2097152, - 'jpg' => 2097152, - 'jpeg' => 2097152, - 'png' => 2097152, - 'gif' => 2097152, - 'xml' => 2097152, - 'pdf' => 2097152, - ); - - $this->assertEquals($expected,$actual); - - } - - public function testIsAllowedType() - { - - $isAllowedTest = array( - array('path'=> 'somefile.bla', 'expected' => false), // suspicious - array('path'=> e_SYSTEM."filetypes.xml", 'expected' => true), // okay - array('path'=> e_PLUGIN."gallery/images/butterfly.jpg", 'expected' => true), // okay - ); - - foreach($isAllowedTest as $file) - { - $actual = $this->fl->isAllowedType($file['path']); - $this->assertEquals($file['expected'],$actual, "isAllowedType() failed on: ".$file['path']); - } - - } -/* - public function testSend() - { - - } - - public function testFile_size_encode() - { - - } - - public function testMkDir() - { - - } - - public function testGetRemoteContent() - { - - } - - public function testDelete() - { - - } - - public function testGetRemoteFile() - { - - } - - public function test_chMod() - { - - } - - public function testIsValidURL() - { - - } - - public function testGet_dirs() - { - - } - - public function testGetErrorMessage() - { - - } - - public function testCopy() - { - - } - - public function testInitCurl() - { - - } - - public function testScandir() - { - - } - - public function testGetFiletypeLimits() - { - - } -*/ - public function testFile_size_decode() - { - $arr = array( - '1024' => 1024, - '2kb' => 2048, - '1KB' => 1024, - '1M' => 1048576, - '1G' => 1073741824, - '1Gb' => 1073741824, - '1TB' => 1099511627776, - ); - - foreach($arr as $key => $expected) - { - $actual = $this->fl->file_size_decode($key); - $this->assertEquals($expected,$actual, $key." does not equal ".$expected." bytes"); - } - - } -/* - public function testZip() - { - - } - - public function testSetDefaults() - { - - } - - public function testSetMode() - { - - } - - public function testUnzipArchive() - { - - } - - public function testSetFileFilter() - { - - } - - public function testGetErrorCode() - { - - } - - public function testChmod() - { - - } - - public function testSetFileInfo() - { - - }*/ - - public function testGet_file_info() - { - $path = APP_PATH."/e107_web/lib/font-awesome/4.7.0/fonts/fontawesome-webfont.svg"; - - $ret = $this->fl->get_file_info($path); - - $this->assertEquals('image/svg+xml',$ret['mime']); - - - } -/* - public function testPrepareDirectory() - { - - } - - public function testGetFileExtension() - { - - } - - public function testRmtree() - { - - } - - public function testGet_files() - { - - } - - public function testGetUserDir() - { - - } - - public function testRemoveDir() - { - - } - - public function testUnzipGithubArchive() - { - - } - - public function testGetRootFolder() - { - - } - - public function testGetUploaded() - { - - } - - public function testGitPull() - { - - } - - public function testCleanFileName() - { - - }*/ } + + protected function _after() + { + unlink($this->exploitFile); + unlink($this->filetypesFile); + } + + + public function testIsClean() + { + + $isCleanTest = array( + array('path'=>$this->exploitFile, 'expected' => false), // suspicious + array('path'=>e_SYSTEM."filetypes.xml", 'expected' => true), // okay + array('path'=>e_PLUGIN."gallery/images/butterfly.jpg", 'expected' => true), // okay + ); + + foreach($isCleanTest as $file) + { + $actual = $this->fl->isClean($file['path'], $file['path']); + $this->assertEquals($file['expected'],$actual, "isClean() failed on {$file['path']} with error code: ".$this->fl->getErrorCode()); + } + + } + + public function testGetAllowedFileTypes() + { + $actual = $this->fl->getAllowedFileTypes(); + + $expected = array ( + 'zip' => 2097152, // 2M in bytes + 'gz' => 2097152, + 'jpg' => 2097152, + 'jpeg' => 2097152, + 'png' => 2097152, + 'gif' => 2097152, + 'xml' => 2097152, + 'pdf' => 2097152, + ); + + $this->assertEquals($expected,$actual); + + } + + public function testIsAllowedType() + { + + $isAllowedTest = array( + array('path'=> 'somefile.bla', 'expected' => false), // suspicious + array('path'=> e_SYSTEM."filetypes.xml", 'expected' => true), // okay + array('path'=> e_PLUGIN."gallery/images/butterfly.jpg", 'expected' => true), // okay + ); + + foreach($isAllowedTest as $file) + { + $actual = $this->fl->isAllowedType($file['path']); + $this->assertEquals($file['expected'],$actual, "isAllowedType() failed on: ".$file['path']); + } + + } + /* + public function testSend() + { + + } + + public function testFile_size_encode() + { + + } + + public function testMkDir() + { + + } + + public function testGetRemoteContent() + { + + } + + public function testDelete() + { + + } + + public function testGetRemoteFile() + { + + } + + public function test_chMod() + { + + } + + public function testIsValidURL() + { + + } + + public function testGet_dirs() + { + + } + + public function testGetErrorMessage() + { + + } + + public function testCopy() + { + + } + + public function testInitCurl() + { + + } + + public function testScandir() + { + + } + + public function testGetFiletypeLimits() + { + + } + */ + public function testFile_size_decode() + { + $arr = array( + '1024' => 1024, + '2kb' => 2048, + '1KB' => 1024, + '1M' => 1048576, + '1G' => 1073741824, + '1Gb' => 1073741824, + '1TB' => 1099511627776, + ); + + foreach($arr as $key => $expected) + { + $actual = $this->fl->file_size_decode($key); + $this->assertEquals($expected,$actual, $key." does not equal ".$expected." bytes"); + } + + } + /* + public function testZip() + { + + } + + public function testSetDefaults() + { + + } + + public function testSetMode() + { + + } + + public function testUnzipArchive() + { + + } + + public function testSetFileFilter() + { + + } + + public function testGetErrorCode() + { + + } + + public function testChmod() + { + + } + + public function testSetFileInfo() + { + + }*/ + + public function testGet_file_info() + { + $path = APP_PATH."/e107_web/lib/font-awesome/4.7.0/fonts/fontawesome-webfont.svg"; + + $ret = $this->fl->get_file_info($path); + + $this->assertEquals('image/svg+xml',$ret['mime']); + + + } + /* + public function testPrepareDirectory() + { + + } + + public function testGetFileExtension() + { + + } + + public function testRmtree() + { + + } + + public function testGet_files() + { + + } + + public function testGetUserDir() + { + + } + + public function testRemoveDir() + { + + } + */ + + public function testUnzipGithubArchive() + { + $prefix = 'e107-master'; + $fake_e107_files = [ + 'desired' => [ + '/index.php', + '/e107_admin/index.html', + '/e107_core/index.html', + '/e107_docs/index.html', + '/e107_handlers/index.html', + '/e107_images/index.html', + '/e107_languages/index.html', + '/e107_media/index.html', + '/e107_plugins/index.html', + '/e107_system/index.html', + '/e107_themes/index.html', + '/e107_web/index.html', + ], + 'undesired' => [ + '/.github/codecov.yml', + '/e107_tests/index.php', + '/.codeclimate.yml', + '/.editorconfig', + '/.gitignore', + '/.gitmodules', + '/CONTRIBUTING.md', + '/LICENSE', + '/README.md', + '/install.php', + '/favicon.ico', + ] + ]; + + $src_dest_map = array( + '/e107_admin/' => '/'.e107::getFolder('ADMIN'), + '/e107_core/' => '/'.e107::getFolder('CORE'), + '/e107_docs/' => '/'.e107::getFolder('DOCS'), + '/e107_handlers/' => '/'.e107::getFolder('HANDLERS'), + '/e107_images/' => '/'.e107::getFolder('IMAGES'), + '/e107_languages/' => '/'.e107::getFolder('LANGUAGES'), + '/e107_media/' => '/'.e107::getFolder('MEDIA'), + '/e107_plugins/' => '/'.e107::getFolder('PLUGINS'), + '/e107_system/' => '/'.e107::getFolder('SYSTEM'), + '/e107_themes/' => '/'.e107::getFolder('THEMES'), + '/e107_web/' => '/'.e107::getFolder('WEB'), + ); + + /** + * @var e_file + */ + $e_file = $this->make('e_file', [ + 'getRemoteFile' => function($remote_url, $local_file, $type='temp') use ($fake_e107_files, $prefix) + { + touch(e_TEMP.$local_file); + $archive = new ZipArchive(); + $archive->open(e_TEMP.$local_file, ZipArchive::OVERWRITE); + array_walk_recursive($fake_e107_files, function($fake_filename) use ($archive, $prefix) + { + $archive->addFromString($prefix.$fake_filename, $fake_filename); + }); + $archive->close(); + } + ]); + $destination = e_TEMP."fake-git-remote-destination/"; + mkdir($destination); + $results = $e_file->unzipGithubArchive('core', $destination); + + $this->assertEmpty($results['error'], "Errors not expected from Git remote update"); + $extraction_mapping = array_flip(e107::getInstance()->e107_dirs); + foreach($fake_e107_files['desired'] as $desired_filename) + { + foreach ($src_dest_map as $src => $dest) + { + $desired_filename = preg_replace("/^".preg_quote($src, '/')."/", $dest, $desired_filename); + } + $this->assertContains(realpath($destination.$desired_filename), $results['success'], + "Desired file did not appear in file system"); + } + foreach($fake_e107_files['undesired'] as $undesired_filename) + { + $this->assertContains($prefix.$undesired_filename, $results['skipped']); + } + } + + /* + public function testGetRootFolder() + { + + } + + public function testGetUploaded() + { + + } + + public function testGitPull() + { + + } + + public function testCleanFileName() + { + + }*/ +}