From caa0826c5e34786373ac8972f741a42651339a7d Mon Sep 17 00:00:00 2001 From: Nick Liu Date: Fri, 20 Mar 2020 15:42:45 -0500 Subject: [PATCH] CoreImage: Replaced JSON with SQLite Looks like SQLite scales better when there are more files to put in the database. --- .github/workflows/build-release/CoreImage.php | 148 +++++++----------- 1 file changed, 53 insertions(+), 95 deletions(-) diff --git a/.github/workflows/build-release/CoreImage.php b/.github/workflows/build-release/CoreImage.php index 64f9e2139..037b78029 100644 --- a/.github/workflows/build-release/CoreImage.php +++ b/.github/workflows/build-release/CoreImage.php @@ -12,14 +12,28 @@ require_once("OsHelper.php"); class CoreImage { + /** @var PDO */ + protected $db; + public function __construct($exportFolder, $currentVersion, $imageFile) { set_time_limit(240); - $this->create_image($exportFolder, $currentVersion, $imageFile); + file_put_contents($imageFile, ''); + $this->db = new PDO("sqlite:{$imageFile}"); + $this->db->exec(' + CREATE TABLE IF NOT EXISTS file_hashes ( + path TEXT, + release_version TEXT, + hash TEXT'/*.', + UNIQUE(path, hash) ON CONFLICT IGNORE'*/ . ' + ); + '); + + $this->create_image($exportFolder, $currentVersion); } - function create_image($exportFolder, $currentVersion, $imageFile) + function create_image($exportFolder, $currentVersion) { $data = "generateCurrentChecksums($exportFolder, $currentVersion); + $this->generateCurrentChecksums($exportFolder, $currentVersion); echo("[Core-Image] Scanning Removed Files from Git" . "\n"); - $result = $this->generateRemovedChecksums($carry); - - $json_result = json_encode($result, JSON_PRETTY_PRINT); - $json_string_result = var_export($json_result, true); - $data .= '$core_image = ' . $json_string_result . ';'; - - $fp = fopen($imageFile, 'w'); - fwrite($fp, $data); + $this->generateRemovedChecksums(); } protected function generateCurrentChecksums($exportFolder, $currentVersion) @@ -59,7 +66,9 @@ class CoreImage if (!is_dir($absoluteBase)) return false; $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder)); - $checksums = []; + + $insert_statement = $this->insert_statement($relativePath, $currentVersion, $checksum); + $this->db->beginTransaction(); /** * @var $file DirectoryIterator @@ -73,14 +82,10 @@ class CoreImage if (empty($relativePath) || $relativePath == $absolutePath) continue; $checksum = $this->checksumPath($absolutePath); - $item = self::array_get($checksums, $relativePath, []); - if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum; - self::array_set($checksums, $relativePath, $item); + $insert_statement->execute(); } - ksort($checksums); - - return $checksums; + $this->db->commit(); } protected function checksumPath($filename) @@ -93,79 +98,8 @@ class CoreImage return md5(str_replace(array(chr(13), chr(10)), '', $body)); } - /** - * Get an item from an array using "slash" notation. - * - * Based on Illuminate\Support\Arr::get() - * - * @param array $array - * @param string $key - * @param mixed $default - * @return mixed - * @copyright Copyright (c) Taylor Otwell - * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License - */ - private static function array_get($array, $key, $default = null) + protected function generateRemovedChecksums() { - if (is_null($key)) return $array; - - if (isset($array[$key])) return $array[$key]; - - foreach (explode('/', $key) as $segment) { - if (!is_array($array) || !array_key_exists($segment, $array)) { - return $default; - } - - $array = $array[$segment]; - } - - return $array; - } - - /** - * Set an array item to a given value using "slash" notation. - * - * If no key is given to the method, the entire array will be replaced. - * - * Based on Illuminate\Support\Arr::set() - * - * @param array $array - * @param string|null $key - * @param mixed $value - * @return array - * @copyright Copyright (c) Taylor Otwell - * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License - */ - private static function array_set(&$array, $key, $value) - { - if (is_null($key)) { - return $array = $value; - } - - $keys = explode('/', $key); - - while (count($keys) > 1) { - $key = array_shift($keys); - - // If the key doesn't exist at this depth, we will just create an empty array - // to hold the next value, allowing us to create the arrays to hold final - // values at the correct depth. Then we'll keep digging into the array. - if (!isset($array[$key]) || !is_array($array[$key])) { - $array[$key] = []; - } - - $array = &$array[$key]; - } - - $array[array_shift($keys)] = $value; - - return $array; - } - - protected function generateRemovedChecksums($carry = []) - { - $checksums = $carry; - $stdout = ''; OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout); $tags = explode("\n", trim($stdout)); @@ -182,9 +116,13 @@ class CoreImage return !preg_match("/[a-z]/i", $version); }); + $insert_statement = $this->insert_statement($removedFilePath, $version, $checksum); + $check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash'); + $this->db->beginTransaction(); + foreach ($tags as $tag => $version) { OsHelper::runValidated( - 'git --no-pager diff --name-only --diff-filter D ' . escapeshellarg($tag), + 'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag), $stdout ); $removedFiles = explode("\n", trim($stdout)); @@ -194,11 +132,31 @@ class CoreImage $stdout ); $checksum = $this->checksum($stdout); - $item = self::array_get($checksums, $removedFilePath, []); - if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum; - self::array_set($checksums, $removedFilePath, $item); + $check_statement->execute([':path' => $removedFilePath, ':hash' => $checksum]); + if ($check_statement->fetchColumn() == 0) $insert_statement->execute(); } } - return $checksums; + + $this->db->commit(); + } + + /** + * @param $relativePath + * @param $releaseVersion + * @param $checksum + * @return PDOStatement + */ + private function insert_statement(&$relativePath, &$releaseVersion, &$checksum) + { + $relativePath = $relativePath ?: null; + $releaseVersion = $releaseVersion ?: null; + $checksum = $checksum ?: null; + $insert_statement = $this->db->prepare( + "INSERT INTO file_hashes (path, release_version, hash) VALUES (:path, :release_version, :hash)" + ); + $insert_statement->bindParam(":path", $relativePath); + $insert_statement->bindParam(":release_version", $releaseVersion); + $insert_statement->bindParam(":hash", $checksum); + return $insert_statement; } } \ No newline at end of file