1
0
mirror of https://github.com/e107inc/e107.git synced 2025-08-06 06:38:00 +02:00

CoreImage: API rework

This commit is contained in:
Nick Liu
2020-03-23 22:40:28 -05:00
parent 76bb9f6ad8
commit b425b4e45d
7 changed files with 331 additions and 323 deletions

View File

@@ -0,0 +1,160 @@
<?php
/**
* e107 website system
*
* Copyright (C) 2008-2020 e107 Inc (e107.org)
* Released under the terms and conditions of the
* GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
*/
abstract class CoreImage
{
protected function create_image($exportFolder, $tempFolder, $currentVersion)
{
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$this->generateCurrentChecksums($exportFolder, $currentVersion);
echo("[Core-Image] Scanning Removed Files from Git" . "\n");
$this->generateRemovedChecksums($tempFolder);
}
protected function generateCurrentChecksums($exportFolder, $currentVersion)
{
$absoluteBase = realpath($exportFolder);
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
/**
* @var $file DirectoryIterator
*/
foreach ($iterator as $file)
{
if ($file->isDir()) continue;
$absolutePath = $file->getRealPath();
$relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath);
if (empty($relativePath) || $relativePath == $absolutePath) continue;
$checksum = $this->checksumPath($absolutePath);
$this->insertChecksumIntoDatabase($relativePath, $checksum, $currentVersion);
}
}
protected function checksumPath($filename)
{
return $this->checksum(file_get_contents($filename));
}
protected function checksum($body)
{
return md5(str_replace(array(chr(13), chr(10)), '', $body));
}
abstract protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$releaseVersion);
protected function generateRemovedChecksums($tempFolder)
{
$tags = $this->getGitTags();
$timeMachineFolder = $this->prepTimeMachine($tempFolder);
$this->generateRemovedChecksumsFromTags($tags, $timeMachineFolder);
}
/**
* @return array
*/
protected function getGitTags()
{
$stdout = '';
OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
$tags = explode("\n", trim($stdout));
$versions = [];
foreach ($tags as $tag)
{
$versions[] = preg_replace("/^v/", "", $tag);
}
$tags = array_combine($tags, $versions);
unset($versions);
uasort($tags, function ($a, $b)
{
return -version_compare($a, $b);
});
$tags = array_filter($tags, function ($version)
{
return !preg_match("/[a-z]/i", $version);
});
return $tags;
}
/**
* @param $tempFolder
* @param $repoFolder
* @return string
*/
protected function prepTimeMachine($tempFolder)
{
$timeMachineFolder = $tempFolder . "/git_time_machine/";
OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder));
OsHelper::runValidated('git rev-parse --show-toplevel', $repoFolder);
$repoFolder = realpath(trim($repoFolder) . "/.git");
OsHelper::runValidated(
'cp -a ' .
escapeshellarg($repoFolder) .
' ' .
escapeshellarg($timeMachineFolder)
);
return $timeMachineFolder;
}
/**
* @param array $tags
* @param $timeMachineFolder
* @return mixed
*/
protected function generateRemovedChecksumsFromTags($tags, $timeMachineFolder)
{
foreach ($tags as $tag => $version)
{
$stdout = '';
OsHelper::runValidated(
'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
$stdout
);
$removedFiles = explode("\n", trim($stdout));
OsHelper::runValidated(
'git -C ' . escapeshellarg($timeMachineFolder) . ' ' .
'checkout ' . escapeshellarg($tag)
);
foreach ($removedFiles as $removedFilePath)
{
$checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath);
$this->insertChecksumIntoDatabase($removedFilePath, $checksum, $version);
}
}
OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder));
}
protected function generateStub()
{
$data = "<?php\n";
$data .= "/*\n";
$data .= "+ ----------------------------------------------------------------------------+\n";
$data .= "| e107 website system\n";
$data .= "|\n";
$data .= "| Copyright (C) 2008-" . date("Y") . " e107 Inc. \n";
$data .= "| http://e107.org\n";
// $data .= "| jalist@e107.org\n";
$data .= "|\n";
$data .= "| Released under the terms and conditions of the\n";
$data .= "| GNU General Public License (http://gnu.org).\n";
$data .= "|\n";
$data .= "| \$URL$\n";
$data .= "| \$Id$\n";
$data .= "+----------------------------------------------------------------------------+\n";
$data .= "*/\n\n";
$data .= "if (!defined('e107_INIT')) { exit; }\n\n";
return $data;
}
}

View File

@@ -9,23 +9,17 @@
*/
require_once("OsHelper.php");
require_once("CoreImage.php");
class JsonCoreImage
class JsonCoreImage extends CoreImage
{
private $checksums = [];
public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
$this->create_image($exportFolder, $tempFolder, $currentVersion, $imageFile);
}
$this->create_image($exportFolder, $tempFolder, $currentVersion);
function create_image($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$carry = $this->generateCurrentChecksums($exportFolder, $currentVersion);
echo("[Core-Image] Scanning Removed Files from Git" . "\n");
$result = $this->generateRemovedChecksums($tempFolder, $carry);
$json_result = json_encode($result, JSON_PRETTY_PRINT);
$json_result = json_encode($this->checksums, JSON_PRETTY_PRINT);
$json_string_result = var_export($json_result, true);
$data = $this->generateStub();
$data .= '$core_image = ' . $json_string_result . ';';
@@ -36,102 +30,24 @@ class JsonCoreImage
protected function generateCurrentChecksums($exportFolder, $currentVersion)
{
$absoluteBase = realpath($exportFolder);
if (!is_dir($absoluteBase)) return false;
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
$checksums = [];
/**
* @var $file DirectoryIterator
*/
foreach ($iterator as $file)
{
if ($file->isDir()) continue;
$absolutePath = $file->getRealPath();
$relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath);
if (empty($relativePath) || $relativePath == $absolutePath) continue;
$checksum = $this->checksumPath($absolutePath);
$item = self::array_get($checksums, $relativePath, []);
if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum;
self::array_set($checksums, $relativePath, $item);
}
ksort($checksums);
return $checksums;
parent::generateCurrentChecksums($exportFolder, $currentVersion);
ksort($this->checksums);
}
protected function checksumPath($filename)
protected function generateRemovedChecksums($tempFolder)
{
return $this->checksum(file_get_contents($filename));
parent::generateRemovedChecksums($tempFolder);
ksort($this->checksums);
}
protected function checksum($body)
/**
* @inheritDoc
*/
protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$version)
{
return md5(str_replace(array(chr(13), chr(10)), '', $body));
}
protected function generateRemovedChecksums($tempFolder, $carry)
{
$checksums = $carry;
$stdout = '';
OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
$tags = explode("\n", trim($stdout));
$versions = [];
foreach ($tags as $tag)
{
$versions[] = preg_replace("/^v/", "", $tag);
}
$tags = array_combine($tags, $versions);
unset($versions);
uasort($tags, function ($a, $b)
{
return -version_compare($a, $b);
});
$tags = array_filter($tags, function ($version)
{
return !preg_match("/[a-z]/i", $version);
});
$timeMachineFolder = $tempFolder . "/git_time_machine/";
OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder));
OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder);
$repo_folder = realpath(trim($repo_folder) . "/.git");
OsHelper::runValidated(
'cp -a ' .
escapeshellarg($repo_folder) .
' ' .
escapeshellarg($timeMachineFolder)
);
foreach ($tags as $tag => $version)
{
OsHelper::runValidated(
'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
$stdout
);
$removedFiles = explode("\n", trim($stdout));
OsHelper::runValidated(
'git -C ' . escapeshellarg($timeMachineFolder) . ' ' .
'checkout ' . escapeshellarg($tag)
);
foreach ($removedFiles as $removedFilePath)
{
$checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath);
$item = self::array_get($checksums, $removedFilePath, []);
if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum;
self::array_set($checksums, $removedFilePath, $item);
}
}
OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder));
ksort($checksums);
return $checksums;
$item = self::array_get($this->checksums, $relativePath, []);
if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum;
self::array_set($this->checksums, $relativePath, $item);
}
/**
@@ -207,27 +123,4 @@ class JsonCoreImage
return $array;
}
private function generateStub()
{
$data = "<?php\n";
$data .= "/*\n";
$data .= "+ ----------------------------------------------------------------------------+\n";
$data .= "| e107 website system\n";
$data .= "|\n";
$data .= "| Copyright (C) 2008-" . date("Y") . " e107 Inc. \n";
$data .= "| http://e107.org\n";
// $data .= "| jalist@e107.org\n";
$data .= "|\n";
$data .= "| Released under the terms and conditions of the\n";
$data .= "| GNU General Public License (http://gnu.org).\n";
$data .= "|\n";
$data .= "| \$URL$\n";
$data .= "| \$Id$\n";
$data .= "+----------------------------------------------------------------------------+\n";
$data .= "*/\n\n";
$data .= "if (!defined('e107_INIT')) { exit; }\n\n";
return $data;
}
}

View File

@@ -9,21 +9,35 @@
*/
require_once("OsHelper.php");
require_once("CoreImage.php");
class SqlpharCoreImage
class SqlpharCoreImage extends CoreImage
{
/** @var PDO */
protected $db;
/** @var PDO */
protected $db;
/**
* @var PDOStatement
*/
private $insert_statement;
/**
* @var PDOStatement
*/
private $check_statement;
public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
$imagePharFile = "$imageFile.phar";
$phar = new Phar($imagePharFile);
// Insert bindings
private $relativePath;
private $releaseVersion;
private $checksum;
$imageSqliteFile = "$imageFile.sqlite";
file_put_contents($imageSqliteFile, '');
$this->db = new PDO("sqlite:{$imageSqliteFile}");
$this->db->exec('
public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile)
{
$imagePharFile = "$imageFile.phar";
$phar = new Phar($imagePharFile);
$imageSqliteFile = "$imageFile.sqlite";
file_put_contents($imageSqliteFile, '');
$this->db = new PDO("sqlite:{$imageSqliteFile}");
$this->db->exec('
CREATE TABLE IF NOT EXISTS file_hashes (
path TEXT,
release_version INTEGER,
@@ -31,187 +45,104 @@ class SqlpharCoreImage
UNIQUE(path, hash) ON CONFLICT IGNORE'*/ . '
);
');
$this->db->exec('
$this->db->exec('
CREATE TABLE IF NOT EXISTS versions (
version_id INTEGER PRIMARY KEY,
version_string TEXT,
UNIQUE(version_string) ON CONFLICT IGNORE
);
');
# Retrieval:
# SELECT file_hashes.path, versions.version_string, file_hashes.hash
# FROM file_hashes
# LEFT JOIN versions ON versions.version_id = file_hashes.release_version
# ORDER BY path ASC;
# Retrieval:
# SELECT file_hashes.path, versions.version_string, file_hashes.hash
# FROM file_hashes
# LEFT JOIN versions ON versions.version_id = file_hashes.release_version
# ORDER BY path ASC;
$this->create_image($exportFolder, $tempFolder, $currentVersion);
$this->check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash');
$this->insert_statement = $this->bind_insert(
$this->relativePath,
$this->releaseVersion,
$this->checksum);
$phar->startBuffering();
$phar->setStub($this->generateStub());
$phar->addFile($imageSqliteFile, "core_image.sqlite");
$phar->compressFiles(Phar::BZ2);
$phar->stopBuffering();
rename($imagePharFile, $imageFile);
}
$this->create_image($exportFolder, $tempFolder, $currentVersion);
function create_image($exportFolder, $tempFolder, $currentVersion)
{
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$this->generateCurrentChecksums($exportFolder, $currentVersion);
$phar->startBuffering();
$phar->setStub($this->generateStub());
$phar->addFile($imageSqliteFile, "core_image.sqlite");
$phar->compressFiles(Phar::BZ2);
$phar->stopBuffering();
rename($imagePharFile, $imageFile);
}
echo("[Core-Image] Scanning Removed Files from Git" . "\n");
$this->generateRemovedChecksums($tempFolder);
}
protected function generateCurrentChecksums($exportFolder, $currentVersion)
{
$this->db->beginTransaction();
$this->insert_version($currentVersion);
parent::generateCurrentChecksums($exportFolder, $currentVersion);
$this->db->commit();
}
protected function generateCurrentChecksums($exportFolder, $currentVersion)
{
$absoluteBase = realpath($exportFolder);
if (!is_dir($absoluteBase)) return false;
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
$insert_statement = $this->insert_statement($relativePath, $currentVersion, $checksum);
$this->db->beginTransaction();
$this->insert_version($currentVersion);
/**
* @var $file DirectoryIterator
*/
foreach ($iterator as $file) {
if ($file->isDir()) continue;
$absolutePath = $file->getRealPath();
$relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath);
if (empty($relativePath) || $relativePath == $absolutePath) continue;
$checksum = $this->checksumPath($absolutePath);
$insert_statement->execute();
}
$this->db->commit();
}
protected function checksumPath($filename)
{
return $this->checksum(file_get_contents($filename));
}
protected function checksum($body)
{
return md5(str_replace(array(chr(13), chr(10)), '', $body));
}
protected function generateRemovedChecksums($tempFolder)
{
$stdout = '';
OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
$tags = explode("\n", trim($stdout));
$versions = [];
foreach ($tags as $tag) {
$versions[] = preg_replace("/^v/", "", $tag);
}
$tags = array_combine($tags, $versions);
unset($versions);
uasort($tags, function ($a, $b) {
return -version_compare($a, $b);
});
$tags = array_filter($tags, function ($version) {
return !preg_match("/[a-z]/i", $version);
});
$timeMachineFolder = $tempFolder . "/git_time_machine/";
OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder));
OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder);
$repo_folder = realpath(trim($repo_folder) . "/.git");
OsHelper::runValidated(
'cp -a ' .
escapeshellarg($repo_folder) .
' ' .
escapeshellarg($timeMachineFolder)
);
$insert_statement = $this->insert_statement($removedFilePath, $version, $checksum);
$check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash');
$this->db->beginTransaction();
foreach ($tags as $tag => $version) {
$this->insert_version($version);
OsHelper::runValidated(
'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
$stdout
);
$removedFiles = explode("\n", trim($stdout));
OsHelper::runValidated(
'git -C ' . escapeshellarg($timeMachineFolder) . ' ' .
'checkout ' . escapeshellarg($tag)
);
foreach ($removedFiles as $removedFilePath) {
$checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath);
$check_statement->execute([':path' => $removedFilePath, ':hash' => $checksum]);
if ($check_statement->fetchColumn() == 0) $insert_statement->execute();
}
}
OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder));
$this->db->commit();
}
/**
* @param $relativePath
* @param $releaseVersion
* @param $checksum
* @return PDOStatement
*/
private function insert_statement(&$relativePath, &$releaseVersion, &$checksum)
{
$relativePath = $relativePath ?: null;
$releaseVersion = $releaseVersion ?: null;
$checksum = $checksum ?: null;
$insert_statement = $this->db->prepare(
"INSERT INTO file_hashes (
/**
* @param $relativePath
* @param $releaseVersion
* @param $checksum
* @return PDOStatement
*/
private function bind_insert(&$relativePath, &$releaseVersion, &$checksum)
{
$relativePath = $relativePath ?: null;
$releaseVersion = $releaseVersion ?: null;
$checksum = $checksum ?: null;
$insert_statement = $this->db->prepare(
"INSERT INTO file_hashes (
path, release_version, hash
) VALUES (
:path, (SELECT version_id FROM versions WHERE version_string = :release_version), :hash
)"
);
$insert_statement->bindParam(":path", $relativePath);
$insert_statement->bindParam(":release_version", $releaseVersion);
$insert_statement->bindParam(":hash", $checksum);
return $insert_statement;
}
);
$insert_statement->bindParam(":path", $relativePath);
$insert_statement->bindParam(":release_version", $releaseVersion);
$insert_statement->bindParam(":hash", $checksum);
return $this->insert_statement = $insert_statement;
}
private function insert_version($releaseVersion)
{
$statement = $this->db->prepare(
"INSERT INTO versions (version_id, version_string) VALUES (NULL, ?)"
);
$statement->execute([$releaseVersion]);
}
private function insert_version($releaseVersion)
{
$statement = $this->db->prepare(
"INSERT INTO versions (version_id, version_string) VALUES (NULL, ?)"
);
$statement->execute([$releaseVersion]);
}
private function generateStub()
{
$data = "<?php\n";
$data .= "/*\n";
$data .= "+ ----------------------------------------------------------------------------+\n";
$data .= "| e107 website system\n";
$data .= "|\n";
$data .= "| Copyright (C) 2008-" . date("Y") . " e107 Inc. \n";
$data .= "| http://e107.org\n";
// $data .= "| jalist@e107.org\n";
$data .= "|\n";
$data .= "| Released under the terms and conditions of the\n";
$data .= "| GNU General Public License (http://gnu.org).\n";
$data .= "|\n";
$data .= "| \$URL$\n";
$data .= "| \$Id$\n";
$data .= "+----------------------------------------------------------------------------+\n";
$data .= "*/\n\n";
$data .= "if (!defined('e107_INIT')) { exit; }\n\n";
$data .= "__HALT_COMPILER();";
protected function generateRemovedChecksums($tempFolder)
{
$this->db->beginTransaction();
$tags = $this->getGitTags();
foreach ($tags as $tag => $version)
{
$this->insert_version($version);
}
parent::generateRemovedChecksums($tempFolder);
$this->db->commit();
}
return $data;
}
protected function generateStub()
{
$data = parent::generateStub();
$data .= "__HALT_COMPILER();";
return $data;
}
/**
* @inheritDoc
*/
protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$releaseVersion)
{
$this->relativePath = $relativePath;
$this->checksum = $checksum;
$this->releaseVersion = $releaseVersion;
$this->check_statement->execute([':path' => $relativePath, ':hash' => $checksum]);
if ($this->check_statement->fetchColumn() == 0) $this->insert_statement->execute();
}
}

View File

@@ -67,7 +67,7 @@ class e107Build
exec("git describe --tags", $output, $rc);
$gitVersion = array_pop($output);
$verFileVersion = self::getVerFileVersion($this->gitDir . "/e107_admin/ver.php");
$this->version = self::gitVersionToPhpVersion($gitVersion, $verFileVersion);
$this->version = OsHelper::gitVersionToPhpVersion($gitVersion, $verFileVersion);
$this->validateReadme();
}
@@ -113,19 +113,6 @@ class e107Build
return '0';
}
private static function gitVersionToPhpVersion($gitVersion, $verFileVersion)
{
$verFileVersion = array_shift(explode(" ", $verFileVersion));
$version = preg_replace("/^v/", "", $gitVersion);
$versionSplit = explode("-", $version);
if (count($versionSplit) > 1)
{
if (version_compare($verFileVersion, $versionSplit[0], '>')) $versionSplit[0] = $verFileVersion;
$versionSplit[0] .= "dev";
}
return implode("-", $versionSplit);
}
private function validateReadme()
{
//check for readme files associated with configured releases
@@ -249,7 +236,8 @@ class e107Build
$zipExportFile = 'release_' . $c . ".zip";
$this->gitArchive($zipExportFile, $rel['since']);
$since = isset($rel['since']) ? $rel['since'] : null;
$this->gitArchive($zipExportFile, $since);
$this->gitArchiveUnzip($zipExportFile);

View File

@@ -16,12 +16,33 @@ require_once("e_file_inspector_interface.php");
*/
abstract class e_file_inspector implements e_file_inspector_interface
{
protected $database;
protected $currentVersion;
private $validatedBitmask;
protected $defaultDirsCache;
protected $customDirsCache;
/**
* e_file_inspector constructor
* @param string $database The database from which integrity data may be read or to which integrity data may be
* written. This should be an URL or absolute file path for most implementations.
*/
public function __construct($database)
{
$this->database = $database;
$this->loadDatabase();
}
/**
* Prepare the provided database for reading or writing
*
* Should tolerate a non-existent database and try to create it if a write operation is executed.
*
* @return void
*/
abstract public function loadDatabase();
/**
* Check the integrity of the provided path
*

View File

@@ -17,11 +17,21 @@ class e_file_inspector_json extends e_file_inspector
* @param $jsonFilePath string Absolute path to the file inspector database
*/
public function __construct($jsonFilePath = null)
{
parent::__construct($jsonFilePath);
}
/**
* @inheritDoc
*/
public function loadDatabase()
{
global $core_image;
if ($jsonFilePath === null) $jsonFilePath = e_ADMIN . "core_image.php";
require($jsonFilePath);
$this->coreImage = self::array_slash(json_decode($core_image, true));
@include($this->database);
$this->coreImage = json_decode($core_image, true);
if (!is_array($this->coreImage)) $this->coreImage = [];
$this->coreImage = self::array_slash($this->coreImage);
if (!$this->coreImage) $this->coreImage = [];
unset($core_image);
}

View File

@@ -14,13 +14,19 @@ class e_file_inspector_sqlphar extends e_file_inspector
private $coreImage;
/**
* e_file_inspector_sqlphar constructor.
* @param $pharFilePath string Absolute path to the file inspector database
*/
public function __construct($pharFilePath = null)
{
if ($pharFilePath === null) $pharFilePath = e_ADMIN . "core_image.php";
Phar::loadPhar($pharFilePath, "core_image.phar");
parent::__construct($pharFilePath);
}
/**
* @inheritDoc
*/
public function loadDatabase()
{
Phar::loadPhar($this->database, "core_image.phar");
$tmpFile = tmpfile();
$tmpFilePath = stream_get_meta_data($tmpFile)['uri'];
$this->copyUrlToResource("phar://core_image.phar/core_image.sqlite", $tmpFile);
@@ -84,7 +90,6 @@ class e_file_inspector_sqlphar extends e_file_inspector
}
/**
* Copy file to destination with low memory footprint
* @param $source string URL of the source