mirror/php-e107
1
0
Fork 0
mirror of https://github.com/e107inc/e107.git synced 2025-08-08 15:46:44 +02:00
Code Issues Releases Wiki Activity

CoreImage: Replaced JSON with SQLite

Browse Source 
Looks like SQLite scales better when there are more files to put in the
database.
This commit is contained in:
Nick Liu
2020-03-20 15:42:45 -05:00
parent 20dbdefb77
commit caa0826c5e
1 changed files with 53 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
.github/workflows/build-release/CoreImage.php vendored
View File

@@ -12,14 +12,28 @@ require_once("OsHelper.php");
class CoreImage
{
/** @var PDO */
protected $db;
public function __construct($exportFolder, $currentVersion, $imageFile)
{
set_time_limit(240);
$this->create_image($exportFolder, $currentVersion, $imageFile);
file_put_contents($imageFile, '');
$this->db = new PDO("sqlite:{$imageFile}");
$this->db->exec('
CREATE TABLE IF NOT EXISTS file_hashes (
path TEXT,
release_version TEXT,
hash TEXT'/*.',
UNIQUE(path, hash) ON CONFLICT IGNORE'*/ . '
);
');
$this->create_image($exportFolder, $currentVersion);
}
function create_image($exportFolder, $currentVersion, $imageFile)
function create_image($exportFolder, $currentVersion)
{
$data = "<?php\n";
$data .= "/*\n";
@@ -40,17 +54,10 @@ class CoreImage
$data .= "if (!defined('e107_INIT')) { exit; }\n\n";
echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
$carry = $this->generateCurrentChecksums($exportFolder, $currentVersion);
$this->generateCurrentChecksums($exportFolder, $currentVersion);
echo("[Core-Image] Scanning Removed Files from Git" . "\n");
$result = $this->generateRemovedChecksums($carry);
$json_result = json_encode($result, JSON_PRETTY_PRINT);
$json_string_result = var_export($json_result, true);
$data .= '$core_image = ' . $json_string_result . ';';
$fp = fopen($imageFile, 'w');
fwrite($fp, $data);
$this->generateRemovedChecksums();
}
protected function generateCurrentChecksums($exportFolder, $currentVersion)
@@ -59,7 +66,9 @@ class CoreImage
if (!is_dir($absoluteBase)) return false;
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
$checksums = [];
$insert_statement = $this->insert_statement($relativePath, $currentVersion, $checksum);
$this->db->beginTransaction();
/**
* @var $file DirectoryIterator
@@ -73,14 +82,10 @@ class CoreImage
if (empty($relativePath) || $relativePath == $absolutePath) continue;
$checksum = $this->checksumPath($absolutePath);
$item = self::array_get($checksums, $relativePath, []);
if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum;
self::array_set($checksums, $relativePath, $item);
$insert_statement->execute();
}
ksort($checksums);
return $checksums;
$this->db->commit();
}
protected function checksumPath($filename)
@@ -93,79 +98,8 @@ class CoreImage
return md5(str_replace(array(chr(13), chr(10)), '', $body));
}
/**
* Get an item from an array using "slash" notation.
*
* Based on Illuminate\Support\Arr::get()
*
* @param array $array
* @param string $key
* @param mixed $default
* @return mixed
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_get($array, $key, $default = null)
protected function generateRemovedChecksums()
{
if (is_null($key)) return $array;
if (isset($array[$key])) return $array[$key];
foreach (explode('/', $key) as $segment) {
if (!is_array($array) || !array_key_exists($segment, $array)) {
return $default;
}
$array = $array[$segment];
}
return $array;
}
/**
* Set an array item to a given value using "slash" notation.
*
* If no key is given to the method, the entire array will be replaced.
*
* Based on Illuminate\Support\Arr::set()
*
* @param array $array
* @param string|null $key
* @param mixed $value
* @return array
* @copyright Copyright (c) Taylor Otwell
* @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
*/
private static function array_set(&$array, $key, $value)
{
if (is_null($key)) {
return $array = $value;
}
$keys = explode('/', $key);
while (count($keys) > 1) {
$key = array_shift($keys);
// If the key doesn't exist at this depth, we will just create an empty array
// to hold the next value, allowing us to create the arrays to hold final
// values at the correct depth. Then we'll keep digging into the array.
if (!isset($array[$key]) || !is_array($array[$key])) {
$array[$key] = [];
}
$array = &$array[$key];
}
$array[array_shift($keys)] = $value;
return $array;
}
protected function generateRemovedChecksums($carry = [])
{
$checksums = $carry;
$stdout = '';
OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
$tags = explode("\n", trim($stdout));
@@ -182,9 +116,13 @@ class CoreImage
return !preg_match("/[a-z]/i", $version);
});
$insert_statement = $this->insert_statement($removedFilePath, $version, $checksum);
$check_statement = $this->db->prepare('SELECT COUNT(*) FROM file_hashes WHERE path = :path AND hash = :hash');
$this->db->beginTransaction();
foreach ($tags as $tag => $version) {
OsHelper::runValidated(
'git --no-pager diff --name-only --diff-filter D ' . escapeshellarg($tag),
'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
$stdout
);
$removedFiles = explode("\n", trim($stdout));
@@ -194,11 +132,31 @@ class CoreImage
$stdout
);
$checksum = $this->checksum($stdout);
$item = self::array_get($checksums, $removedFilePath, []);
if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum;
self::array_set($checksums, $removedFilePath, $item);
$check_statement->execute([':path' => $removedFilePath, ':hash' => $checksum]);
if ($check_statement->fetchColumn() == 0) $insert_statement->execute();
}
}
return $checksums;
$this->db->commit();
}
/**
* @param $relativePath
* @param $releaseVersion
* @param $checksum
* @return PDOStatement
*/
private function insert_statement(&$relativePath, &$releaseVersion, &$checksum)
{
$relativePath = $relativePath ?: null;
$releaseVersion = $releaseVersion ?: null;
$checksum = $checksum ?: null;
$insert_statement = $this->db->prepare(
"INSERT INTO file_hashes (path, release_version, hash) VALUES (:path, :release_version, :hash)"
);
$insert_statement->bindParam(":path", $relativePath);
$insert_statement->bindParam(":release_version", $releaseVersion);
$insert_statement->bindParam(":hash", $checksum);
return $insert_statement;
}
}