Restored JSON core image generator for performance

PDO SQLite was performing unacceptably poorly.
Let's just load the entire integrity image into RAM…

.github/workflows/build-release/JsonCoreImage.php

@@ -0,0 +1,233 @@
+<?php
+/**
+ * e107 website system
+ *
+ * Copyright (C) 2008-2020 e107 Inc (e107.org)
+ * Released under the terms and conditions of the
+ * GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
+ *
+ */
+
+require_once("OsHelper.php");
+
+class JsonCoreImage
+{
+    public function __construct($exportFolder, $tempFolder, $currentVersion, $imageFile)
+    {
+        $this->create_image($exportFolder, $tempFolder, $currentVersion, $imageFile);
+    }
+
+    function create_image($exportFolder, $tempFolder, $currentVersion, $imageFile)
+    {
+        echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
+        $carry = $this->generateCurrentChecksums($exportFolder, $currentVersion);
+
+        echo("[Core-Image] Scanning Removed Files from Git" . "\n");
+        $result = $this->generateRemovedChecksums($tempFolder, $carry);
+
+        $json_result = json_encode($result, JSON_PRETTY_PRINT);
+        $json_string_result = var_export($json_result, true);
+        $data = $this->generateStub();
+        $data .= '$core_image = ' . $json_string_result . ';';
+
+        $fp = fopen($imageFile, 'w');
+        fwrite($fp, $data);
+    }
+
+    protected function generateCurrentChecksums($exportFolder, $currentVersion)
+    {
+        $absoluteBase = realpath($exportFolder);
+        if (!is_dir($absoluteBase)) return false;
+
+        $iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($exportFolder));
+        $checksums = [];
+
+        /**
+         * @var $file DirectoryIterator
+         */
+        foreach ($iterator as $file)
+        {
+            if ($file->isDir()) continue;
+
+            $absolutePath = $file->getRealPath();
+            $relativePath = preg_replace("/^" . preg_quote($absoluteBase . "/", "/") . "/", "", $absolutePath);
+
+            if (empty($relativePath) || $relativePath == $absolutePath) continue;
+
+            $checksum = $this->checksumPath($absolutePath);
+            $item = self::array_get($checksums, $relativePath, []);
+            if (!in_array($checksum, $item)) $item["v{$currentVersion}"] = $checksum;
+            self::array_set($checksums, $relativePath, $item);
+        }
+
+        ksort($checksums);
+        return $checksums;
+    }
+
+    protected function checksumPath($filename)
+    {
+        return $this->checksum(file_get_contents($filename));
+    }
+
+    protected function checksum($body)
+    {
+        return md5(str_replace(array(chr(13), chr(10)), '', $body));
+    }
+
+    protected function generateRemovedChecksums($tempFolder, $carry)
+    {
+        $checksums = $carry;
+
+        $stdout = '';
+        OsHelper::runValidated('git tag --list ' . escapeshellarg("v*"), $stdout);
+        $tags = explode("\n", trim($stdout));
+        $versions = [];
+        foreach ($tags as $tag)
+        {
+            $versions[] = preg_replace("/^v/", "", $tag);
+        }
+        $tags = array_combine($tags, $versions);
+        unset($versions);
+        uasort($tags, function ($a, $b)
+        {
+            return -version_compare($a, $b);
+        });
+        $tags = array_filter($tags, function ($version)
+        {
+            return !preg_match("/[a-z]/i", $version);
+        });
+
+        $timeMachineFolder = $tempFolder . "/git_time_machine/";
+        OsHelper::runValidated('mkdir -p ' . escapeshellarg($timeMachineFolder));
+        OsHelper::runValidated('git rev-parse --show-toplevel', $repo_folder);
+        $repo_folder = realpath(trim($repo_folder) . "/.git");
+        OsHelper::runValidated(
+            'cp -a ' .
+            escapeshellarg($repo_folder) .
+            ' ' .
+            escapeshellarg($timeMachineFolder)
+        );
+
+        foreach ($tags as $tag => $version)
+        {
+            OsHelper::runValidated(
+                'git --no-pager diff --no-renames --name-only --diff-filter D ' . escapeshellarg($tag),
+                $stdout
+            );
+            $removedFiles = explode("\n", trim($stdout));
+            OsHelper::runValidated(
+                'git -C ' . escapeshellarg($timeMachineFolder) . ' ' .
+                'checkout ' . escapeshellarg($tag)
+            );
+            foreach ($removedFiles as $removedFilePath)
+            {
+                $checksum = $this->checksumPath($timeMachineFolder . '/' . $removedFilePath);
+                $item = self::array_get($checksums, $removedFilePath, []);
+                if (!in_array($checksum, $item)) $item["v{$version}"] = $checksum;
+                self::array_set($checksums, $removedFilePath, $item);
+            }
+        }
+
+        OsHelper::runValidated('rm -rf ' . escapeshellarg($timeMachineFolder));
+
+        ksort($checksums);
+        return $checksums;
+    }
+
+    /**
+     * Get an item from an array using "slash" notation.
+     *
+     * Based on Illuminate\Support\Arr::get()
+     *
+     * @param array $array
+     * @param string $key
+     * @param mixed $default
+     * @return mixed
+     * @copyright Copyright (c) Taylor Otwell
+     * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
+     */
+    private static function array_get($array, $key, $default = null)
+    {
+        if (is_null($key)) return $array;
+
+        if (isset($array[$key])) return $array[$key];
+
+        foreach (explode('/', $key) as $segment)
+        {
+            if (!is_array($array) || !array_key_exists($segment, $array))
+            {
+                return $default;
+            }
+
+            $array = $array[$segment];
+        }
+
+        return $array;
+    }
+
+    /**
+     * Set an array item to a given value using "slash" notation.
+     *
+     * If no key is given to the method, the entire array will be replaced.
+     *
+     * Based on Illuminate\Support\Arr::set()
+     *
+     * @param array $array
+     * @param string|null $key
+     * @param mixed $value
+     * @return array
+     * @copyright Copyright (c) Taylor Otwell
+     * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
+     */
+    private static function array_set(&$array, $key, $value)
+    {
+        if (is_null($key))
+        {
+            return $array = $value;
+        }
+
+        $keys = explode('/', $key);
+
+        while (count($keys) > 1)
+        {
+            $key = array_shift($keys);
+
+            // If the key doesn't exist at this depth, we will just create an empty array
+            // to hold the next value, allowing us to create the arrays to hold final
+            // values at the correct depth. Then we'll keep digging into the array.
+            if (!isset($array[$key]) || !is_array($array[$key]))
+            {
+                $array[$key] = [];
+            }
+
+            $array = &$array[$key];
+        }
+
+        $array[array_shift($keys)] = $value;
+
+        return $array;
+    }
+
+    private function generateStub()
+    {
+        $data = "<?php\n";
+        $data .= "/*\n";
+        $data .= "+ ----------------------------------------------------------------------------+\n";
+        $data .= "|     e107 website system\n";
+        $data .= "|\n";
+        $data .= "|     Copyright (C) 2008-" . date("Y") . " e107 Inc. \n";
+        $data .= "|     http://e107.org\n";
+        //	$data .= "|     jalist@e107.org\n";
+        $data .= "|\n";
+        $data .= "|     Released under the terms and conditions of the\n";
+        $data .= "|     GNU General Public License (http://gnu.org).\n";
+        $data .= "|\n";
+        $data .= "|     \$URL$\n";
+        $data .= "|     \$Id$\n";
+        $data .= "+----------------------------------------------------------------------------+\n";
+        $data .= "*/\n\n";
+        $data .= "if (!defined('e107_INIT')) { exit; }\n\n";
+
+        return $data;
+    }
+}

.github/workflows/build-release/SqlpharCoreImage.php

@@ -10,7 +10,7 @@
 
 require_once("OsHelper.php");
 
-class CoreImage
+class SqlpharCoreImage
 {
 	/** @var PDO */
 	protected $db;
@@ -56,6 +56,7 @@ class CoreImage
 
 	function create_image($exportFolder, $tempFolder, $currentVersion)
 	{
+        echo("[Core-Image] Scanning Dir: " . $exportFolder . "\n");
 		$this->generateCurrentChecksums($exportFolder, $currentVersion);
 
 		echo("[Core-Image] Scanning Removed Files from Git" . "\n");

.github/workflows/build-release/e107_make.php

@@ -9,7 +9,7 @@
  */
 
 require_once("OsHelper.php");
-require_once("CoreImage.php");
+require_once("JsonCoreImage.php");
 
 $builder = new e107Build();
 $builder->makeBuild();
@@ -517,7 +517,7 @@ class e107Build
 		$imageFile = $this->tempDir . "core_image.php";
 
 		$this->status("Creating new core_image.php file ({$imageFile})", true);
-		new CoreImage($this->exportDir, $this->tempDir, $this->version, $imageFile);
+		new JsonCoreImage($this->exportDir, $this->tempDir, $this->version, $imageFile);
 
 		$dir = "{$this->config['baseDir']}/target/{$this->config['main']['name']}/export";
 		$this->changeDir($dir);

class2.php

@@ -2033,7 +2033,7 @@ e107::getDebug()->log("Timezone: ".USERTIMEZONE); // remove later on.
 		define('USERNAME', 'e107-cli');
 		define('USERTHEME', false);
 		define('ADMIN', true);
-		define('ADMINPERMS', false);
+		define('ADMINPERMS', '0');
 		define('GUEST', false);
 		define('USERCLASS', '');
 		define('USEREMAIL', '');

e107_handlers/e_file_inspector.php

@@ -16,8 +16,12 @@ require_once("e_file_inspector_interface.php");
  */
 abstract class e_file_inspector implements e_file_inspector_interface
 {
+    protected $currentVersion;
     private $validatedBitmask;
 
+    protected $defaultDirsCache;
+    protected $customDirsCache;
+
     /**
      * Check the integrity of the provided path
      *
@@ -105,10 +109,12 @@ abstract class e_file_inspector implements e_file_inspector_interface
      */
     public function getCurrentVersion()
     {
+        if ($this->currentVersion) return $this->currentVersion;
+
         $checksums = $this->getChecksums("index.php");
         $versions = array_keys($checksums);
         usort($versions, 'version_compare');
-        return array_pop($versions);
+        return $this->currentVersion = array_pop($versions);
     }
 
     /**
@@ -138,6 +144,27 @@ abstract class e_file_inspector implements e_file_inspector_interface
         return false;
     }
 
+    protected function pathToDefaultPath($path)
+    {
+        if (!$this->customDirsCache)
+        {
+            $this->defaultDirsCache = e107::getInstance()->defaultDirs();
+            $customDirs = e107::getInstance()->e107_dirs ? e107::getInstance()->e107_dirs : [];
+            $this->customDirsCache = array_diff_assoc($customDirs, $this->defaultDirsCache);
+        }
+        foreach ($this->customDirsCache as $dirType => $customDir)
+        {
+            if (!isset($this->defaultDirsCache[$dirType])) continue;
+
+            $defaultDir = $this->defaultDirsCache[$dirType];
+            if ($customDir === $defaultDir) continue;
+
+            if (substr($path, 0, strlen($customDir)) === $customDir)
+                $path = $defaultDir . substr($path, strlen($customDir));
+        }
+        return $path;
+    }
+
     private function getValidatedBitmask()
     {
         if ($this->validatedBitmask !== null) return $this->validatedBitmask;

e107_handlers/e_file_inspector_json.php

@@ -0,0 +1,114 @@
+<?php
+/**
+ * e107 website system
+ *
+ * Copyright (C) 2008-2020 e107 Inc (e107.org)
+ * Released under the terms and conditions of the
+ * GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
+ */
+
+require_once("e_file_inspector.php");
+
+class e_file_inspector_json extends e_file_inspector
+{
+    private $coreImage;
+
+    /**
+     * @param $jsonFilePath string Absolute path to the file inspector database
+     */
+    public function __construct($jsonFilePath = null)
+    {
+        global $core_image;
+        if ($jsonFilePath === null) $jsonFilePath = e_ADMIN . "core_image.php";
+        require($jsonFilePath);
+        $this->coreImage = json_decode($core_image, true);
+        unset($core_image);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function getPathIterator($version = null)
+    {
+        $result = self::array_slash($this->coreImage);
+        if (!empty($version))
+        {
+            $result = array_filter($result, function ($value) use ($version)
+            {
+                return array_key_exists($version, $value);
+            });
+        }
+        return new ArrayIterator(array_keys($result));
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function getChecksums($path)
+    {
+        $path = $this->pathToDefaultPath($path);
+        return self::array_get($this->coreImage, $path, []);
+    }
+
+    /**
+     * Get an item from an array using "slash" notation.
+     *
+     * Based on Illuminate\Support\Arr::get()
+     *
+     * @param array $array
+     * @param string $key
+     * @param mixed $default
+     * @return mixed
+     * @copyright Copyright (c) Taylor Otwell
+     * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
+     */
+    private static function array_get($array, $key, $default = null)
+    {
+        if (is_null($key)) return $array;
+
+        if (isset($array[$key])) return $array[$key];
+
+        foreach (explode('/', $key) as $segment)
+        {
+            if (!is_array($array) || !array_key_exists($segment, $array))
+            {
+                return $default;
+            }
+
+            $array = $array[$segment];
+        }
+
+        return $array;
+    }
+
+    /**
+     * Flatten a multi-dimensional associative array with slashes.
+     * Excludes the second-to-last level of depth from flattening.
+     *
+     * Based on Illuminate\Support\Arr::dot()
+     *
+     * @param array $array
+     * @param string $prepend
+     * @return array
+     * @copyright Copyright (c) Taylor Otwell
+     * @license https://github.com/illuminate/support/blob/master/LICENSE.md MIT License
+     */
+    private static function array_slash($array, $prepend = '')
+    {
+        $results = array();
+
+        foreach ($array as $key => $value)
+        {
+            if (is_array($value) && is_array(reset($value)))
+            {
+                $results = array_merge($results, self::array_slash($value, $prepend . $key . '/'));
+            }
+            else
+            {
+                $results[$prepend . $key] = $value;
+            }
+        }
+
+        return $results;
+    }
+}

e107_handlers/e_file_inspector_sqlphar.php

@@ -12,10 +12,6 @@ require_once("e_file_inspector.php");
 class e_file_inspector_sqlphar extends e_file_inspector
 {
     private $coreImage;
-    private $currentVersion;
-
-    private $defaultDirsCache;
-    private $customDirsCache;
 
     /**
      * e_file_inspector_sqlphar constructor.
@@ -87,26 +83,7 @@ class e_file_inspector_sqlphar extends e_file_inspector
         return $this->currentVersion = $statement->fetchColumn();
     }
 
-    private function pathToDefaultPath($path)
-    {
-        if (!$this->customDirsCache)
-        {
-            $this->defaultDirsCache = e107::getInstance()->defaultDirs();
-            $customDirs = e107::getInstance()->e107_dirs ? e107::getInstance()->e107_dirs : [];
-            $this->customDirsCache = array_diff_assoc($customDirs, $this->defaultDirsCache);
-        }
-        foreach ($this->customDirsCache as $dirType => $customDir)
-        {
-            if (!isset($this->defaultDirsCache[$dirType])) continue;
 
-            $defaultDir = $this->defaultDirsCache[$dirType];
-            if ($customDir === $defaultDir) continue;
-
-            if (substr($path, 0, strlen($customDir)) === $customDir)
-                $path = $defaultDir . substr($path, strlen($customDir));
-        }
-        return $path;
-    }
 
     /**
      * Copy file to destination with low memory footprint

e107_tests/tests/unit/e_file_inspectorTest.php

@@ -16,8 +16,9 @@ class e_file_inspectorTest extends \Codeception\Test\Unit
 
     public function _before()
     {
+        require_once(e_HANDLER . "e_file_inspector_json.php");
         require_once(e_HANDLER . "e_file_inspector_sqlphar.php");
-        $this->e_integrity = new e_file_inspector_sqlphar();
+        $this->e_integrity = new e_file_inspector_json();
     }
 
     public function testGetChecksums()
@@ -69,7 +70,7 @@ class e_file_inspectorTest extends \Codeception\Test\Unit
      */
     public function testPathToDefaultPath()
     {
-        $object = new e_file_inspector_sqlphar();
+        $object = $this->make('e_file_inspector');
         $class = new ReflectionClass(get_class($object));
         $method = $class->getMethod('pathToDefaultPath');
         $method->setAccessible(true);