mirror/php-e107
1
0
Fork 0
mirror of https://github.com/e107inc/e107.git synced 2025-07-31 11:50:30 +02:00
Code Issues Releases Wiki Activity

Restored JSON core image generator for performance

Browse Source 
PDO SQLite was performing unacceptably poorly.
Let's just load the entire integrity image into RAM…
This commit is contained in:
Nick Liu
2020-03-23 17:04:44 -05:00
parent 90bdc88d55
commit b8ed70693f
9 changed files with 383 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
.github/workflows/build-release/JsonCoreImage.php vendored Normal file
View File

@@ -0,0 +1,233 @@
<?php
/**
* e107 website system
*
* Copyright (C) 2008-2020 e107 Inc (e107.org)
* Released under the terms and conditions of the
* GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
*
*/
require_once("OsHelper.php");
class JsonCoreImage
{
public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
$this->create_image($exportFolder, $tempFolder, $currentVersion, $imageFile);
}
function create_image($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$carry = $this->generateCurrentChecksums($exportFolder, $currentVersion);
echo("[Core-Image] Scanning Removed Files from Git" . "\n");
$result = $this->generateRemovedChecksums($tempFolder, $carry);
$json_result = json_encode($result, JSON_PRETTY_PRINT);
$json_string_result = var_export($json_result, true);
$data = $this->generateStub();
$data .= '$core_image = ' . $json_string_result . ';';
$fp = fopen($imageFile, 'w');
fwrite($fp, $data);
}
protected function generateCurrentChecksums($exportFolder, $currentVersion)
{
$absoluteBase = realpath($exportFolder);
if (!is_dir($absoluteBase)) return false;
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
$checksums = [];
/**
* @var $file DirectoryIterator
*/
foreach ($iterator as $file)
{
if ($file->isDir()) continue;
$absolutePath = $file->getRealPath();
$relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath);
if (empty($relativePath) || $relativePath == $absolutePath) continue;
$checksum = $this->checksumPath($absolutePath);
$item = self::array_get($checksums, $relativePath, []);
if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum;
self::array_set($checksums, $relativePath, $item);
}
ksort($checksums);
return $checksums;
}
protected function checksumPath($filename)
{
return $this->checksum(file_get_contents($filename));
}
protected function checksum($body)
{
return md5(str_replace(array(chr(13), chr(10)), '', $body));
}
protected function generateRemovedChecksums($tempFolder, $carry)
{
$checksums = $carry;
$stdout = '';
OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
$tags = explode("\n", trim($stdout));
$versions = [];
foreach ($tags as $tag)
{
$versions[] = preg_replace("/^v/", "", $tag);
}
$tags = array_combine($tags, $versions);
unset($versions);
uasort($tags, function ($a, $b)
{
return -version_compare($a, $b);
});
$tags = array_filter($tags, function ($version)
{
return !preg_match("/[a-z]/i", $version);
});
$timeMachineFolder = $tempFolder . "/git_time_machine/";
OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder));
OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder);
$repo_folder = realpath(trim($repo_folder) . "/.git");
OsHelper::runValidated(
'cp -a ' .
escapeshellarg($repo_folder) .
' ' .
escapeshellarg($timeMachineFolder)
);
foreach ($tags as $tag => $version)
{
OsHelper::runValidated(
'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
$stdout
);
$removedFiles = explode("\n", trim($stdout));
OsHelper::runValidated(
'git -C ' . escapeshellarg($timeMachineFolder) . ' ' .
'checkout ' . escapeshellarg($tag)
);
foreach ($removedFiles as $removedFilePath)
{
$checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath);
$item = self::array_get($checksums, $removedFilePath, []);
if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum;
self::array_set($checksums, $removedFilePath, $item);
}
}
OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder));
ksort($checksums);
return $checksums;
}
/**
* Get an item from an array using "slash" notation.
*
* Based on Illuminate\Support\Arr::get()
*
* @param array $array
* @param string $key
* @param mixed $default
* @return mixed
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_get($array, $key, $default = null)
{
if (is_null($key)) return $array;
if (isset($array[$key])) return $array[$key];
foreach (explode('/', $key) as $segment)
{
if (!is_array($array) || !array_key_exists($segment, $array))
{
return $default;
}
$array = $array[$segment];
}
return $array;
}
/**
* Set an array item to a given value using "slash" notation.
*
* If no key is given to the method, the entire array will be replaced.
*
* Based on Illuminate\Support\Arr::set()
*
* @param array $array
* @param string|null $key
* @param mixed $value
* @return array
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_set(&$array, $key, $value)
{
if (is_null($key))
{
return $array = $value;
}
$keys = explode('/', $key);
while (count($keys) > 1)
{
$key = array_shift($keys);
// If the key doesn't exist at this depth, we will just create an empty array
// to hold the next value, allowing us to create the arrays to hold final
// values at the correct depth. Then we'll keep digging into the array.
if (!isset($array[$key]) || !is_array($array[$key]))
{
$array[$key] = [];
}
$array = &$array[$key];
}
$array[array_shift($keys)] = $value;
return $array;
}
private function generateStub()
{
$data = "<?php\n";
$data .= "/*\n";
$data .= "+ ----------------------------------------------------------------------------+\n";
$data .= "| e107 website system\n";
$data .= "|\n";
$data .= "| Copyright (C) 2008-" . date("Y") . " e107 Inc. \n";
$data .= "| http://e107.org\n";
// $data .= "| jalist@e107.org\n";
$data .= "|\n";
$data .= "| Released under the terms and conditions of the\n";
$data .= "| GNU General Public License (http://gnu.org).\n";
$data .= "|\n";
$data .= "| \$URL$\n";
$data .= "| \$Id$\n";
$data .= "+----------------------------------------------------------------------------+\n";
$data .= "*/\n\n";
$data .= "if (!defined('e107_INIT')) { exit; }\n\n";
return $data;
}
}

3
.github/workflows/build-release/CoreImage.php → .github/workflows/build-release/SqlpharCoreImage.php vendored
View File

@@ -10,7 +10,7 @@
require_once("OsHelper.php");
class CoreImage
class SqlpharCoreImage
{
/** @var PDO */
protected $db;
@@ -56,6 +56,7 @@ class CoreImage
function create_image($exportFolder, $tempFolder, $currentVersion)
{
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$this->generateCurrentChecksums($exportFolder, $currentVersion);
echo("[Core-Image] Scanning Removed Files from Git" . "\n");

4
.github/workflows/build-release/e107_make.php vendored
View File

@@ -9,7 +9,7 @@
*/
require_once("OsHelper.php");
require_once("CoreImage.php");
require_once("JsonCoreImage.php");
$builder = new e107Build();
$builder->makeBuild();
@@ -517,7 +517,7 @@ class e107Build
$imageFile = $this->tempDir . "core_image.php";
$this->status("Creating new core_image.php file ({$imageFile})", true);
new CoreImage($this->exportDir, $this->tempDir, $this->version, $imageFile);
new JsonCoreImage($this->exportDir, $this->tempDir, $this->version, $imageFile);
$dir = "{$this->config['baseDir']}/target/{$this->config['main']['name']}/export";
$this->changeDir($dir);

2
class2.php
View File

@@ -2033,7 +2033,7 @@ e107::getDebug()->log("Timezone: ".USERTIMEZONE); // remove later on.
define('USERNAME', 'e107-cli');
define('USERTHEME', false);
define('ADMIN', true);
define('ADMINPERMS', false);
define('ADMINPERMS', '0');
define('GUEST', false);
define('USERCLASS', '');
define('USEREMAIL', '');

BIN
e107_admin/core_image.php
View File

Binary file not shown.

29
e107_handlers/e_file_inspector.php
View File

@@ -16,8 +16,12 @@ require_once("e_file_inspector_interface.php");
*/
abstract class e_file_inspector implements e_file_inspector_interface
{
protected $currentVersion;
private $validatedBitmask;
protected $defaultDirsCache;
protected $customDirsCache;
/**
* Check the integrity of the provided path
*
@@ -105,10 +109,12 @@ abstract class e_file_inspector implements e_file_inspector_interface
*/
public function getCurrentVersion()
{
if ($this->currentVersion) return $this->currentVersion;
$checksums = $this->getChecksums("index.php");
$versions = array_keys($checksums);
usort($versions, 'version_compare');
return array_pop($versions);
return $this->currentVersion = array_pop($versions);
}
/**
@@ -138,6 +144,27 @@ abstract class e_file_inspector implements e_file_inspector_interface
return false;
}
protected function pathToDefaultPath($path)
{
if (!$this->customDirsCache)
{
$this->defaultDirsCache = e107::getInstance()->defaultDirs();
$customDirs = e107::getInstance()->e107_dirs ? e107::getInstance()->e107_dirs : [];
$this->customDirsCache = array_diff_assoc($customDirs, $this->defaultDirsCache);
}
foreach ($this->customDirsCache as $dirType => $customDir)
{
if (!isset($this->defaultDirsCache[$dirType])) continue;
$defaultDir = $this->defaultDirsCache[$dirType];
if ($customDir === $defaultDir) continue;
if (substr($path, 0, strlen($customDir)) === $customDir)
$path = $defaultDir . substr($path, strlen($customDir));
}
return $path;
}
private function getValidatedBitmask()
{
if ($this->validatedBitmask !== null) return $this->validatedBitmask;

114
e107_handlers/e_file_inspector_json.php Normal file
View File

@@ -0,0 +1,114 @@
<?php
/**
* e107 website system
*
* Copyright (C) 2008-2020 e107 Inc (e107.org)
* Released under the terms and conditions of the
* GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
*/
require_once("e_file_inspector.php");
class e_file_inspector_json extends e_file_inspector
{
private $coreImage;
/**
* @param $jsonFilePath string Absolute path to the file inspector database
*/
public function __construct($jsonFilePath = null)
{
global $core_image;
if ($jsonFilePath === null) $jsonFilePath = e_ADMIN . "core_image.php";
require($jsonFilePath);
$this->coreImage = json_decode($core_image, true);
unset($core_image);
}
/**
* @inheritDoc
*/
public function getPathIterator($version = null)
{
$result = self::array_slash($this->coreImage);
if (!empty($version))
{
$result = array_filter($result, function ($value) use ($version)
{
return array_key_exists($version, $value);
});
}
return new ArrayIterator(array_keys($result));
}
/**
* @inheritDoc
*/
public function getChecksums($path)
{
$path = $this->pathToDefaultPath($path);
return self::array_get($this->coreImage, $path, []);
}
/**
* Get an item from an array using "slash" notation.
*
* Based on Illuminate\Support\Arr::get()
*
* @param array $array
* @param string $key
* @param mixed $default
* @return mixed
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_get($array, $key, $default = null)
{
if (is_null($key)) return $array;
if (isset($array[$key])) return $array[$key];
foreach (explode('/', $key) as $segment)
{
if (!is_array($array) || !array_key_exists($segment, $array))
{
return $default;
}
$array = $array[$segment];
}
return $array;
}
/**
* Flatten a multi-dimensional associative array with slashes.
* Excludes the second-to-last level of depth from flattening.
*
* Based on Illuminate\Support\Arr::dot()
*
* @param array $array
* @param string $prepend
* @return array
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_slash($array, $prepend = '')
{
$results = array();
foreach ($array as $key => $value)
{
if (is_array($value) && is_array(reset($value)))
{
$results = array_merge($results, self::array_slash($value, $prepend . $key . '/'));
}
else
{
$results[$prepend . $key] = $value;
}
}
return $results;
}
}

23
e107_handlers/e_file_inspector_sqlphar.php
View File

@@ -12,10 +12,6 @@ require_once("e_file_inspector.php");
class e_file_inspector_sqlphar extends e_file_inspector
{
private $coreImage;
private $currentVersion;
private $defaultDirsCache;
private $customDirsCache;
/**
* e_file_inspector_sqlphar constructor.
@@ -87,26 +83,7 @@ class e_file_inspector_sqlphar extends e_file_inspector
return $this->currentVersion = $statement->fetchColumn();
}
private function pathToDefaultPath($path)
{
if (!$this->customDirsCache)
{
$this->defaultDirsCache = e107::getInstance()->defaultDirs();
$customDirs = e107::getInstance()->e107_dirs ? e107::getInstance()->e107_dirs : [];
$this->customDirsCache = array_diff_assoc($customDirs, $this->defaultDirsCache);
}
foreach ($this->customDirsCache as $dirType => $customDir)
{
if (!isset($this->defaultDirsCache[$dirType])) continue;
$defaultDir = $this->defaultDirsCache[$dirType];
if ($customDir === $defaultDir) continue;
if (substr($path, 0, strlen($customDir)) === $customDir)
$path = $defaultDir . substr($path, strlen($customDir));
}
return $path;
}
/**
* Copy file to destination with low memory footprint

5
e107_tests/tests/unit/e_file_inspectorTest.php
View File

@@ -16,8 +16,9 @@ class e_file_inspectorTest extends \Codeception\Test\Unit
public function _before()
{
require_once(e_HANDLER . "e_file_inspector_json.php");
require_once(e_HANDLER . "e_file_inspector_sqlphar.php");
$this->e_integrity = new e_file_inspector_sqlphar();
$this->e_integrity = new e_file_inspector_json();
}
public function testGetChecksums()
@@ -69,7 +70,7 @@ class e_file_inspectorTest extends \Codeception\Test\Unit
*/
public function testPathToDefaultPath()
{
$object = new e_file_inspector_sqlphar();
$object = $this->make('e_file_inspector');
$class = new ReflectionClass(get_class($object));
$method = $class->getMethod('pathToDefaultPath');
$method->setAccessible(true);