diff --git a/.github/workflows/build-release/CoreImage.php b/.github/workflows/build-release/CoreImage.php new file mode 100644 index 000000000..41d26339e --- /dev/null +++ b/.github/workflows/build-release/CoreImage.php @@ -0,0 +1,160 @@ +generateCurrentChecksums($exportFolder, $currentVersion); + + echo("[Core-Image] Scanning Removed Files from Git" . "\n"); + $this->generateRemovedChecksums($tempFolder); + } + + protected function generateCurrentChecksums($exportFolder, $currentVersion) + { + $absoluteBase = realpath($exportFolder); + $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder)); + + /** + * @var $file DirectoryIterator + */ + foreach ($iterator as $file) + { + if ($file->isDir()) continue; + + $absolutePath = $file->getRealPath(); + $relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath); + + if (empty($relativePath) || $relativePath == $absolutePath) continue; + + $checksum = $this->checksumPath($absolutePath); + $this->insertChecksumIntoDatabase($relativePath, $checksum, $currentVersion); + } + } + + protected function checksumPath($filename) + { + return $this->checksum(file_get_contents($filename)); + } + + protected function checksum($body) + { + return md5(str_replace(array(chr(13), chr(10)), '', $body)); + } + + abstract protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$releaseVersion); + + protected function generateRemovedChecksums($tempFolder) + { + $tags = $this->getGitTags(); + $timeMachineFolder = $this->prepTimeMachine($tempFolder); + $this->generateRemovedChecksumsFromTags($tags, $timeMachineFolder); + } + + /** + * @return array + */ + protected function getGitTags() + { + $stdout = ''; + OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout); + $tags = explode("\n", trim($stdout)); + $versions = []; + foreach ($tags as $tag) + { + $versions[] = preg_replace("/^v/", "", $tag); + } + $tags = array_combine($tags, $versions); + unset($versions); + uasort($tags, function ($a, $b) + { + return -version_compare($a, $b); + }); + $tags = array_filter($tags, function ($version) + { + return !preg_match("/[a-z]/i", $version); + }); + return $tags; + } + + /** + * @param $tempFolder + * @param $repoFolder + * @return string + */ + protected function prepTimeMachine($tempFolder) + { + $timeMachineFolder = $tempFolder . "/git_time_machine/"; + OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder)); + OsHelper::runValidated('git rev-parse --show-toplevel', $repoFolder); + $repoFolder = realpath(trim($repoFolder) . "/.git"); + OsHelper::runValidated( + 'cp -a ' . + escapeshellarg($repoFolder) . + ' ' . + escapeshellarg($timeMachineFolder) + ); + return $timeMachineFolder; + } + + /** + * @param array $tags + * @param $timeMachineFolder + * @return mixed + */ + protected function generateRemovedChecksumsFromTags($tags, $timeMachineFolder) + { + foreach ($tags as $tag => $version) + { + $stdout = ''; + OsHelper::runValidated( + 'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag), + $stdout + ); + $removedFiles = explode("\n", trim($stdout)); + OsHelper::runValidated( + 'git -C ' . escapeshellarg($timeMachineFolder) . ' ' . + 'checkout ' . escapeshellarg($tag) + ); + foreach ($removedFiles as $removedFilePath) + { + $checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath); + $this->insertChecksumIntoDatabase($removedFilePath, $checksum, $version); + } + } + + OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder)); + } + + + protected function generateStub() + { + $data = "create_image($exportFolder, $tempFolder, $currentVersion, $imageFile); - } + $this->create_image($exportFolder, $tempFolder, $currentVersion); - function create_image($exportFolder, $tempFolder, $currentVersion, $imageFile) - { - echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n"); - $carry = $this->generateCurrentChecksums($exportFolder, $currentVersion); - - echo("[Core-Image] Scanning Removed Files from Git" . "\n"); - $result = $this->generateRemovedChecksums($tempFolder, $carry); - - $json_result = json_encode($result, JSON_PRETTY_PRINT); + $json_result = json_encode($this->checksums, JSON_PRETTY_PRINT); $json_string_result = var_export($json_result, true); $data = $this->generateStub(); $data .= '$core_image = ' . $json_string_result . ';'; @@ -36,102 +30,24 @@ class JsonCoreImage protected function generateCurrentChecksums($exportFolder, $currentVersion) { - $absoluteBase = realpath($exportFolder); - if (!is_dir($absoluteBase)) return false; - - $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder)); - $checksums = []; - - /** - * @var $file DirectoryIterator - */ - foreach ($iterator as $file) - { - if ($file->isDir()) continue; - - $absolutePath = $file->getRealPath(); - $relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath); - - if (empty($relativePath) || $relativePath == $absolutePath) continue; - - $checksum = $this->checksumPath($absolutePath); - $item = self::array_get($checksums, $relativePath, []); - if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum; - self::array_set($checksums, $relativePath, $item); - } - - ksort($checksums); - return $checksums; + parent::generateCurrentChecksums($exportFolder, $currentVersion); + ksort($this->checksums); } - protected function checksumPath($filename) + protected function generateRemovedChecksums($tempFolder) { - return $this->checksum(file_get_contents($filename)); + parent::generateRemovedChecksums($tempFolder); + ksort($this->checksums); } - protected function checksum($body) + /** + * @inheritDoc + */ + protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$version) { - return md5(str_replace(array(chr(13), chr(10)), '', $body)); - } - - protected function generateRemovedChecksums($tempFolder, $carry) - { - $checksums = $carry; - - $stdout = ''; - OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout); - $tags = explode("\n", trim($stdout)); - $versions = []; - foreach ($tags as $tag) - { - $versions[] = preg_replace("/^v/", "", $tag); - } - $tags = array_combine($tags, $versions); - unset($versions); - uasort($tags, function ($a, $b) - { - return -version_compare($a, $b); - }); - $tags = array_filter($tags, function ($version) - { - return !preg_match("/[a-z]/i", $version); - }); - - $timeMachineFolder = $tempFolder . "/git_time_machine/"; - OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder)); - OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder); - $repo_folder = realpath(trim($repo_folder) . "/.git"); - OsHelper::runValidated( - 'cp -a ' . - escapeshellarg($repo_folder) . - ' ' . - escapeshellarg($timeMachineFolder) - ); - - foreach ($tags as $tag => $version) - { - OsHelper::runValidated( - 'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag), - $stdout - ); - $removedFiles = explode("\n", trim($stdout)); - OsHelper::runValidated( - 'git -C ' . escapeshellarg($timeMachineFolder) . ' ' . - 'checkout ' . escapeshellarg($tag) - ); - foreach ($removedFiles as $removedFilePath) - { - $checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath); - $item = self::array_get($checksums, $removedFilePath, []); - if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum; - self::array_set($checksums, $removedFilePath, $item); - } - } - - OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder)); - - ksort($checksums); - return $checksums; + $item = self::array_get($this->checksums, $relativePath, []); + if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum; + self::array_set($this->checksums, $relativePath, $item); } /** @@ -207,27 +123,4 @@ class JsonCoreImage return $array; } - - private function generateStub() - { - $data = "db = new PDO("sqlite:{$imageSqliteFile}"); - $this->db->exec(' + public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile) + { + $imagePharFile = "$imageFile.phar"; + $phar = new Phar($imagePharFile); + + $imageSqliteFile = "$imageFile.sqlite"; + file_put_contents($imageSqliteFile, ''); + $this->db = new PDO("sqlite:{$imageSqliteFile}"); + $this->db->exec(' CREATE TABLE IF NOT EXISTS file_hashes ( path TEXT, release_version INTEGER, @@ -31,187 +45,104 @@ class SqlpharCoreImage UNIQUE(path, hash) ON CONFLICT IGNORE'*/ . ' ); '); - $this->db->exec(' + $this->db->exec(' CREATE TABLE IF NOT EXISTS versions ( version_id INTEGER PRIMARY KEY, version_string TEXT, UNIQUE(version_string) ON CONFLICT IGNORE ); '); - # Retrieval: - # SELECT file_hashes.path, versions.version_string, file_hashes.hash - # FROM file_hashes - # LEFT JOIN versions ON versions.version_id = file_hashes.release_version - # ORDER BY path ASC; + # Retrieval: + # SELECT file_hashes.path, versions.version_string, file_hashes.hash + # FROM file_hashes + # LEFT JOIN versions ON versions.version_id = file_hashes.release_version + # ORDER BY path ASC; - $this->create_image($exportFolder, $tempFolder, $currentVersion); + $this->check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash'); + $this->insert_statement = $this->bind_insert( + $this->relativePath, + $this->releaseVersion, + $this->checksum); - $phar->startBuffering(); - $phar->setStub($this->generateStub()); - $phar->addFile($imageSqliteFile, "core_image.sqlite"); - $phar->compressFiles(Phar::BZ2); - $phar->stopBuffering(); - rename($imagePharFile, $imageFile); - } + $this->create_image($exportFolder, $tempFolder, $currentVersion); - function create_image($exportFolder, $tempFolder, $currentVersion) - { - echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n"); - $this->generateCurrentChecksums($exportFolder, $currentVersion); + $phar->startBuffering(); + $phar->setStub($this->generateStub()); + $phar->addFile($imageSqliteFile, "core_image.sqlite"); + $phar->compressFiles(Phar::BZ2); + $phar->stopBuffering(); + rename($imagePharFile, $imageFile); + } - echo("[Core-Image] Scanning Removed Files from Git" . "\n"); - $this->generateRemovedChecksums($tempFolder); - } + protected function generateCurrentChecksums($exportFolder, $currentVersion) + { + $this->db->beginTransaction(); + $this->insert_version($currentVersion); + parent::generateCurrentChecksums($exportFolder, $currentVersion); + $this->db->commit(); + } - protected function generateCurrentChecksums($exportFolder, $currentVersion) - { - $absoluteBase = realpath($exportFolder); - if (!is_dir($absoluteBase)) return false; - - $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder)); - - $insert_statement = $this->insert_statement($relativePath, $currentVersion, $checksum); - $this->db->beginTransaction(); - - $this->insert_version($currentVersion); - - /** - * @var $file DirectoryIterator - */ - foreach ($iterator as $file) { - if ($file->isDir()) continue; - - $absolutePath = $file->getRealPath(); - $relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath); - - if (empty($relativePath) || $relativePath == $absolutePath) continue; - - $checksum = $this->checksumPath($absolutePath); - $insert_statement->execute(); - } - - $this->db->commit(); - } - - protected function checksumPath($filename) - { - return $this->checksum(file_get_contents($filename)); - } - - protected function checksum($body) - { - return md5(str_replace(array(chr(13), chr(10)), '', $body)); - } - - protected function generateRemovedChecksums($tempFolder) - { - $stdout = ''; - OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout); - $tags = explode("\n", trim($stdout)); - $versions = []; - foreach ($tags as $tag) { - $versions[] = preg_replace("/^v/", "", $tag); - } - $tags = array_combine($tags, $versions); - unset($versions); - uasort($tags, function ($a, $b) { - return -version_compare($a, $b); - }); - $tags = array_filter($tags, function ($version) { - return !preg_match("/[a-z]/i", $version); - }); - - $timeMachineFolder = $tempFolder . "/git_time_machine/"; - OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder)); - OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder); - $repo_folder = realpath(trim($repo_folder) . "/.git"); - OsHelper::runValidated( - 'cp -a ' . - escapeshellarg($repo_folder) . - ' ' . - escapeshellarg($timeMachineFolder) - ); - - $insert_statement = $this->insert_statement($removedFilePath, $version, $checksum); - $check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash'); - $this->db->beginTransaction(); - - foreach ($tags as $tag => $version) { - $this->insert_version($version); - OsHelper::runValidated( - 'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag), - $stdout - ); - $removedFiles = explode("\n", trim($stdout)); - OsHelper::runValidated( - 'git -C ' . escapeshellarg($timeMachineFolder) . ' ' . - 'checkout ' . escapeshellarg($tag) - ); - foreach ($removedFiles as $removedFilePath) { - $checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath); - $check_statement->execute([':path' => $removedFilePath, ':hash' => $checksum]); - if ($check_statement->fetchColumn() == 0) $insert_statement->execute(); - } - } - - OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder)); - - $this->db->commit(); - } - - /** - * @param $relativePath - * @param $releaseVersion - * @param $checksum - * @return PDOStatement - */ - private function insert_statement(&$relativePath, &$releaseVersion, &$checksum) - { - $relativePath = $relativePath ?: null; - $releaseVersion = $releaseVersion ?: null; - $checksum = $checksum ?: null; - $insert_statement = $this->db->prepare( - "INSERT INTO file_hashes ( + /** + * @param $relativePath + * @param $releaseVersion + * @param $checksum + * @return PDOStatement + */ + private function bind_insert(&$relativePath, &$releaseVersion, &$checksum) + { + $relativePath = $relativePath ?: null; + $releaseVersion = $releaseVersion ?: null; + $checksum = $checksum ?: null; + $insert_statement = $this->db->prepare( + "INSERT INTO file_hashes ( path, release_version, hash ) VALUES ( :path, (SELECT version_id FROM versions WHERE version_string = :release_version), :hash )" - ); - $insert_statement->bindParam(":path", $relativePath); - $insert_statement->bindParam(":release_version", $releaseVersion); - $insert_statement->bindParam(":hash", $checksum); - return $insert_statement; - } + ); + $insert_statement->bindParam(":path", $relativePath); + $insert_statement->bindParam(":release_version", $releaseVersion); + $insert_statement->bindParam(":hash", $checksum); + return $this->insert_statement = $insert_statement; + } - private function insert_version($releaseVersion) - { - $statement = $this->db->prepare( - "INSERT INTO versions (version_id, version_string) VALUES (NULL, ?)" - ); - $statement->execute([$releaseVersion]); - } + private function insert_version($releaseVersion) + { + $statement = $this->db->prepare( + "INSERT INTO versions (version_id, version_string) VALUES (NULL, ?)" + ); + $statement->execute([$releaseVersion]); + } - private function generateStub() - { - $data = "db->beginTransaction(); + $tags = $this->getGitTags(); + foreach ($tags as $tag => $version) + { + $this->insert_version($version); + } + parent::generateRemovedChecksums($tempFolder); + $this->db->commit(); + } - return $data; - } + protected function generateStub() + { + $data = parent::generateStub(); + $data .= "__HALT_COMPILER();"; + + return $data; + } + + /** + * @inheritDoc + */ + protected function insertChecksumIntoDatabase(&$relativePath, &$checksum, &$releaseVersion) + { + $this->relativePath = $relativePath; + $this->checksum = $checksum; + $this->releaseVersion = $releaseVersion; + $this->check_statement->execute([':path' => $relativePath, ':hash' => $checksum]); + if ($this->check_statement->fetchColumn() == 0) $this->insert_statement->execute(); + } } \ No newline at end of file diff --git a/.github/workflows/build-release/e107_make.php b/.github/workflows/build-release/e107_make.php index 91f3fb37a..b6f8f39d6 100644 --- a/.github/workflows/build-release/e107_make.php +++ b/.github/workflows/build-release/e107_make.php @@ -67,7 +67,7 @@ class e107Build exec("git describe --tags", $output, $rc); $gitVersion = array_pop($output); $verFileVersion = self::getVerFileVersion($this->gitDir . "/e107_admin/ver.php"); - $this->version = self::gitVersionToPhpVersion($gitVersion, $verFileVersion); + $this->version = OsHelper::gitVersionToPhpVersion($gitVersion, $verFileVersion); $this->validateReadme(); } @@ -113,19 +113,6 @@ class e107Build return '0'; } - private static function gitVersionToPhpVersion($gitVersion, $verFileVersion) - { - $verFileVersion = array_shift(explode(" ", $verFileVersion)); - $version = preg_replace("/^v/", "", $gitVersion); - $versionSplit = explode("-", $version); - if (count($versionSplit) > 1) - { - if (version_compare($verFileVersion, $versionSplit[0], '>')) $versionSplit[0] = $verFileVersion; - $versionSplit[0] .= "dev"; - } - return implode("-", $versionSplit); - } - private function validateReadme() { //check for readme files associated with configured releases @@ -249,7 +236,8 @@ class e107Build $zipExportFile = 'release_' . $c . ".zip"; - $this->gitArchive($zipExportFile, $rel['since']); + $since = isset($rel['since']) ? $rel['since'] : null; + $this->gitArchive($zipExportFile, $since); $this->gitArchiveUnzip($zipExportFile); diff --git a/e107_handlers/e_file_inspector.php b/e107_handlers/e_file_inspector.php index c17e074df..28f7481f1 100644 --- a/e107_handlers/e_file_inspector.php +++ b/e107_handlers/e_file_inspector.php @@ -16,12 +16,33 @@ require_once("e_file_inspector_interface.php"); */ abstract class e_file_inspector implements e_file_inspector_interface { + protected $database; protected $currentVersion; private $validatedBitmask; protected $defaultDirsCache; protected $customDirsCache; + /** + * e_file_inspector constructor + * @param string $database The database from which integrity data may be read or to which integrity data may be + * written. This should be an URL or absolute file path for most implementations. + */ + public function __construct($database) + { + $this->database = $database; + $this->loadDatabase(); + } + + /** + * Prepare the provided database for reading or writing + * + * Should tolerate a non-existent database and try to create it if a write operation is executed. + * + * @return void + */ + abstract public function loadDatabase(); + /** * Check the integrity of the provided path * diff --git a/e107_handlers/e_file_inspector_json.php b/e107_handlers/e_file_inspector_json.php index 3f6f0f4ed..4d1448416 100644 --- a/e107_handlers/e_file_inspector_json.php +++ b/e107_handlers/e_file_inspector_json.php @@ -17,11 +17,21 @@ class e_file_inspector_json extends e_file_inspector * @param $jsonFilePath string Absolute path to the file inspector database */ public function __construct($jsonFilePath = null) + { + parent::__construct($jsonFilePath); + } + + /** + * @inheritDoc + */ + public function loadDatabase() { global $core_image; - if ($jsonFilePath === null) $jsonFilePath = e_ADMIN . "core_image.php"; - require($jsonFilePath); - $this->coreImage = self::array_slash(json_decode($core_image, true)); + @include($this->database); + $this->coreImage = json_decode($core_image, true); + if (!is_array($this->coreImage)) $this->coreImage = []; + $this->coreImage = self::array_slash($this->coreImage); + if (!$this->coreImage) $this->coreImage = []; unset($core_image); } diff --git a/e107_handlers/e_file_inspector_sqlphar.php b/e107_handlers/e_file_inspector_sqlphar.php index d40695654..85b68e474 100644 --- a/e107_handlers/e_file_inspector_sqlphar.php +++ b/e107_handlers/e_file_inspector_sqlphar.php @@ -14,13 +14,19 @@ class e_file_inspector_sqlphar extends e_file_inspector private $coreImage; /** - * e_file_inspector_sqlphar constructor. * @param $pharFilePath string Absolute path to the file inspector database */ public function __construct($pharFilePath = null) { - if ($pharFilePath === null) $pharFilePath = e_ADMIN . "core_image.php"; - Phar::loadPhar($pharFilePath, "core_image.phar"); + parent::__construct($pharFilePath); + } + + /** + * @inheritDoc + */ + public function loadDatabase() + { + Phar::loadPhar($this->database, "core_image.phar"); $tmpFile = tmpfile(); $tmpFilePath = stream_get_meta_data($tmpFile)['uri']; $this->copyUrlToResource("phar://core_image.phar/core_image.sqlite", $tmpFile); @@ -84,7 +90,6 @@ class e_file_inspector_sqlphar extends e_file_inspector } - /** * Copy file to destination with low memory footprint * @param $source string URL of the source