diff --git a/cache/README.md b/cache/README.md
index 40a723d5825..408e014a9f6 100644
--- a/cache/README.md
+++ b/cache/README.md
@@ -9,6 +9,7 @@ A definition:
$definitions = array(
'string' => array( // Required, unique to the component
'mode' => cache_store::MODE_APPLICATION, // Required
+ 'simplekeys' => false, // Optional
'simpledata' => false, // Optional
'requireidentifiers' => array( // Optional
'lang'
@@ -105,6 +106,7 @@ The following settings are required for a definition:
* mode - Application, session or request.
The following optional settings can also be defined:
+* simplekeys - Set to true if items will always and only have simple keys. Simple keys may contain a-zA-Z0-9_. If set to true we use the keys as they are without hashing them. Good for performance and possible because we know the keys are safe.
* simpledata - Set to true if you know that you will only be storing scalar values or arrays of scalar values. Avoids costly investigation of data types.
* requireidentifiers - Any identifiers the definition requires. Must be provided when creating the loader.
* requiredataguarantee - If set to true then only stores that support data guarantee will be used.
diff --git a/cache/classes/definition.php b/cache/classes/definition.php
index dcc219d805a..d02e082a768 100644
--- a/cache/classes/definition.php
+++ b/cache/classes/definition.php
@@ -39,6 +39,11 @@ defined('MOODLE_INTERNAL') || die();
* [int] Sets the mode for the definition. Must be one of cache_store::MODE_*
*
* Optional settings:
+ * + simplekeys
+ * [bool] Set to true if your cache will only use simple keys for its items.
+ * Simple keys consist of digits, underscores and the 26 chars of the english language. a-zA-Z0-9_
+ * If true the keys won't be hashed before being passed to the cache store for gets/sets/deletes. It will be
+ * better for performance and possible only becase we know the keys are safe.
* + simpledata
* [bool] If set to true we know that the data is scalar or array of scalar.
* + requireidentifiers
@@ -129,6 +134,12 @@ class cache_definition {
*/
protected $area;
+ /**
+ * If set to true we know the keys are simple. a-zA-Z0-9_
+ * @var bool
+ */
+ protected $simplekeys = false;
+
/**
* Set to true if we know the data is scalar or array of scalar.
* @var bool
@@ -289,6 +300,7 @@ class cache_definition {
$area = (string)$definition['area'];
// Set the defaults.
+ $simplekeys = false;
$simpledata = false;
$requireidentifiers = array();
$requiredataguarantee = false;
@@ -306,6 +318,9 @@ class cache_definition {
$mappingsonly = false;
$invalidationevents = array();
+ if (array_key_exists('simplekeys', $definition)) {
+ $simplekeys = (bool)$definition['simplekeys'];
+ }
if (array_key_exists('simpledata', $definition)) {
$simpledata = (bool)$definition['simpledata'];
}
@@ -410,6 +425,7 @@ class cache_definition {
$cachedefinition->mode = $mode;
$cachedefinition->component = $component;
$cachedefinition->area = $area;
+ $cachedefinition->simplekeys = $simplekeys;
$cachedefinition->simpledata = $simpledata;
$cachedefinition->requireidentifiers = $requireidentifiers;
$cachedefinition->requiredataguarantee = $requiredataguarantee;
@@ -515,6 +531,17 @@ class cache_definition {
return $this->component;
}
+ /**
+ * Returns true if this definition is using simple keys.
+ *
+ * Simple keys contain only a-zA-Z0-9_
+ *
+ * @return bool
+ */
+ public function uses_simple_keys() {
+ return $this->simplekeys;
+ }
+
/**
* Returns the identifiers that are being used for this definition.
* @return array
diff --git a/cache/classes/factory.php b/cache/classes/factory.php
index 729a46bf6c8..ddf4e8da871 100644
--- a/cache/classes/factory.php
+++ b/cache/classes/factory.php
@@ -164,8 +164,6 @@ class cache_factory {
$definition->set_identifiers($identifiers);
$cache = $this->create_cache($definition, $identifiers);
if ($definition->should_be_persistent()) {
- $cache->persist = true;
- $cache->persistcache = array();
$this->cachesfromparams[$key] = $cache;
}
return $cache;
diff --git a/cache/classes/helper.php b/cache/classes/helper.php
index 9a03556bad4..154d9d86472 100644
--- a/cache/classes/helper.php
+++ b/cache/classes/helper.php
@@ -446,12 +446,18 @@ class cache_helper {
}
/**
- * Hashes a descriptive key to make it shorter and stil unique.
- * @param string $key
+ * Hashes a descriptive key to make it shorter and still unique.
+ * @param string|int $key
+ * @param cache_definition $definition
* @return string
*/
- public static function hash_key($key) {
- return crc32($key);
+ public static function hash_key($key, cache_definition $definition) {
+ if ($definition->uses_simple_keys()) {
+ // We put the key first so that we can be sure the start of the key changes.
+ return (string)$key . '-' . $definition->generate_single_key_prefix();
+ }
+ $key = $definition->generate_single_key_prefix() . '-' . $key;
+ return sha1($key);
}
/**
diff --git a/cache/classes/loaders.php b/cache/classes/loaders.php
index ef7b308d11b..d0b3a4de1e5 100644
--- a/cache/classes/loaders.php
+++ b/cache/classes/loaders.php
@@ -794,12 +794,14 @@ class cache implements cache_loader {
* @return string|array String unless the store supports multi-identifiers in which case an array if returned.
*/
protected function parse_key($key) {
+ // First up if the store supports multiple keys we'll go with that.
if ($this->store->supports_multiple_indentifiers()) {
$result = $this->definition->generate_multi_key_parts();
$result['key'] = $key;
return $result;
}
- return cache_helper::hash_key($this->definition->generate_single_key_prefix().'-'.$key);
+ // If not we need to generate a hash and to for that we use the cache_helper.
+ return cache_helper::hash_key($key, $this->definition);
}
/**
diff --git a/cache/stores/file/addinstanceform.php b/cache/stores/file/addinstanceform.php
index a5e3125e801..18eb3e49b7c 100644
--- a/cache/stores/file/addinstanceform.php
+++ b/cache/stores/file/addinstanceform.php
@@ -52,6 +52,10 @@ class cachestore_file_addinstance_form extends cachestore_addinstance_form {
$form->addHelpButton('autocreate', 'autocreate', 'cachestore_file');
$form->disabledIf('autocreate', 'path', 'eq', '');
+ $form->addElement('checkbox', 'singledirectory', get_string('singledirectory', 'cachestore_file'));
+ $form->setType('singledirectory', PARAM_BOOL);
+ $form->addHelpButton('singledirectory', 'singledirectory', 'cachestore_file');
+
$form->addElement('checkbox', 'prescan', get_string('prescan', 'cachestore_file'));
$form->setType('prescan', PARAM_BOOL);
$form->addHelpButton('prescan', 'prescan', 'cachestore_file');
diff --git a/cache/stores/file/lang/en/cachestore_file.php b/cache/stores/file/lang/en/cachestore_file.php
index 8a17b71f95a..f0097c250c6 100644
--- a/cache/stores/file/lang/en/cachestore_file.php
+++ b/cache/stores/file/lang/en/cachestore_file.php
@@ -35,3 +35,18 @@ $string['path_help'] = 'The directory that should be used to store files for thi
$string['pluginname'] = 'File cache';
$string['prescan'] = 'Prescan directory';
$string['prescan_help'] = 'If enabled the directory is scanned when the cache is first used and requests for files are first checked against the scan data. This can help if you have a slow file system and are finding that file operations are causing you a bottle neck.';
+$string['singledirectory'] = 'Single directory store';
+$string['singledirectory_help'] = 'If enabled files (cached items) will be stored in a single directory rather than being broken up into multiple directories.
+Enabling this will speed up file interactions but comes at the cost of increased risk of hitting file system limitations.
+It is advisable to only turn this on if the following is true:
+ - If you know the number of items in the cache is going to be small enough that it won\'t cause issues on the file system you are running with.
+ - The data being cached is not expensive to generate. If it is then sticking with the default may still be the better option as it reduces the chance of issues.';
+
+/**
+ * This is is like the file store, but designed for siutations where:
+ * - many more things are likely to be stored in the cache, so CRC hashing is
+ * too likely to give collisions, and storing everything in a completely flat
+ * directory structure is inadvisable.
+ * - the things we are caching are more expensive to calculate, so the extra
+ * time to computer a better hash is a worthwhile trade-off.
+ */
\ No newline at end of file
diff --git a/cache/stores/file/lib.php b/cache/stores/file/lib.php
index 03d2b32ffca..3e985122420 100644
--- a/cache/stores/file/lib.php
+++ b/cache/stores/file/lib.php
@@ -57,6 +57,14 @@ class cachestore_file implements cache_store, cache_is_key_aware {
*/
protected $prescan = false;
+ /**
+ * Set to true if we should store files within a single directory.
+ * By default we use a nested structure in order to reduce the chance of conflicts and avoid any file system
+ * limitations such as maximum files per directory.
+ * @var bool
+ */
+ protected $singledirectory = false;
+
/**
* Set to true when the path should be automatically created if it does not yet exist.
* @var bool
@@ -122,7 +130,20 @@ class cachestore_file implements cache_store, cache_is_key_aware {
}
$this->isready = $path !== false;
$this->path = $path;
- $this->prescan = array_key_exists('prescan', $configuration) ? (bool)$configuration['prescan'] : false;
+ // Check if we should prescan the directory.
+ if (array_key_exists('prescan', $configuration)) {
+ $this->prescan = (bool)$configuration['prescan'];
+ } else {
+ // Default is no, we should not prescan.
+ $this->prescan = false;
+ }
+ // Check if we should be storing in a single directory.
+ if (array_key_exists('singledirectory', $configuration)) {
+ $this->singledirectory = (bool)$configuration['singledirectory'];
+ } else {
+ // Default: No, we will use multiple directories.
+ $this->singledirectory = false;
+ }
}
/**
@@ -226,10 +247,51 @@ class cachestore_file implements cache_store, cache_is_key_aware {
$this->prescan = false;
}
if ($this->prescan) {
- $pattern = $this->path.'/*.cache';
- foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) {
- $this->keys[basename($filename)] = filemtime($filename);
+ $this->prescan_keys();
+ }
+ }
+
+ /**
+ * Pre-scan the cache to see which keys are present.
+ */
+ protected function prescan_keys() {
+ foreach (glob($this->glob_keys_pattern(), GLOB_MARK | GLOB_NOSORT) as $filename) {
+ $this->keys[basename($filename)] = filemtime($filename);
+ }
+ }
+
+ /**
+ * Gets a pattern suitable for use with glob to find all keys in the cache.
+ * @return string The pattern.
+ */
+ protected function glob_keys_pattern() {
+ if ($this->singledirectory) {
+ return $this->path . '/*.cache';
+ } else {
+ return $this->path . '/*/*.cache';
+ }
+ }
+
+ /**
+ * Returns the file path to use for the given key.
+ *
+ * @param string $key The key to generate a file path for.
+ * @param bool $create If set to the true the directory structure the key requires will be created.
+ * @return string The full path to the file that stores a particular cache key.
+ */
+ protected function file_path_for_key($key, $create = false) {
+ if ($this->singledirectory) {
+ // Its a single directory, easy, just the store instances path + the file name.
+ return $this->path . '/' . $key . '.cache';
+ } else {
+ // We are using a single subdirectory to achieve 1 level.
+ $subdir = substr($key, 0, 3);
+ $dir = $this->path . '/' . $subdir;
+ if ($create) {
+ // Create the directory. This function does it recursivily!
+ make_writable_directory($dir);
}
+ return $dir . '/' . $key . '.cache';
}
}
@@ -241,7 +303,7 @@ class cachestore_file implements cache_store, cache_is_key_aware {
*/
public function get($key) {
$filename = $key.'.cache';
- $file = $this->path.'/'.$filename;
+ $file = $this->file_path_for_key($key);
$ttl = $this->definition->get_ttl();
if ($ttl) {
$maxtime = cache::now() - $ttl;
@@ -307,7 +369,7 @@ class cachestore_file implements cache_store, cache_is_key_aware {
*/
public function delete($key) {
$filename = $key.'.cache';
- $file = $this->path.'/'.$filename;
+ $file = $this->file_path_for_key($key);
$result = @unlink($file);
unset($this->keys[$filename]);
return $result;
@@ -339,7 +401,7 @@ class cachestore_file implements cache_store, cache_is_key_aware {
public function set($key, $data) {
$this->ensure_path_exists();
$filename = $key.'.cache';
- $file = $this->path.'/'.$filename;
+ $file = $this->file_path_for_key($key, true);
$result = $this->write_file($file, $this->prep_data_before_save($data));
if (!$result) {
// Couldn't write the file.
@@ -404,11 +466,11 @@ class cachestore_file implements cache_store, cache_is_key_aware {
*/
public function has($key) {
$filename = $key.'.cache';
- $file = $this->path.'/'.$key.'.cache';
$maxtime = cache::now() - $this->definition->get_ttl();
if ($this->prescan) {
return array_key_exists($filename, $this->keys) && $this->keys[$filename] >= $maxtime;
}
+ $file = $this->file_path_for_key($key);
return (file_exists($file) && ($this->definition->get_ttl() == 0 || filemtime($file) >= $maxtime));
}
@@ -448,8 +510,7 @@ class cachestore_file implements cache_store, cache_is_key_aware {
* @return boolean True on success. False otherwise.
*/
public function purge() {
- $pattern = $this->path.'/*.cache';
- foreach (glob($pattern, GLOB_MARK | GLOB_NOSORT) as $filename) {
+ foreach (glob($this->glob_keys_pattern(), GLOB_MARK | GLOB_NOSORT) as $filename) {
@unlink($filename);
}
$this->keys = array();
@@ -471,6 +532,9 @@ class cachestore_file implements cache_store, cache_is_key_aware {
if (isset($data->autocreate)) {
$config['autocreate'] = $data->autocreate;
}
+ if (isset($data->singledirectory)) {
+ $config['singledirectory'] = $data->singledirectory;
+ }
if (isset($data->prescan)) {
$config['prescan'] = $data->prescan;
}
diff --git a/cache/tests/cache_test.php b/cache/tests/cache_test.php
index 324d361d6b3..a15a31cc072 100644
--- a/cache/tests/cache_test.php
+++ b/cache/tests/cache_test.php
@@ -364,6 +364,50 @@ class cache_phpunit_tests extends advanced_testcase {
$this->assertEquals('Test has no value really.', $cache->get('Test'));
}
+ /**
+ * Test a very basic definition.
+ */
+ public function test_definition() {
+ $instance = cache_config_phpunittest::instance();
+ $instance->phpunit_add_definition('phpunit/test', array(
+ 'mode' => cache_store::MODE_APPLICATION,
+ 'component' => 'phpunit',
+ 'area' => 'test',
+ ));
+ $cache = cache::make('phpunit', 'test');
+
+ $this->assertTrue($cache->set('testkey1', 'test data 1'));
+ $this->assertEquals('test data 1', $cache->get('testkey1'));
+ $this->assertTrue($cache->set('testkey2', 'test data 2'));
+ $this->assertEquals('test data 2', $cache->get('testkey2'));
+ }
+
+ /**
+ * Test a definition using the simple keys.
+ */
+ public function test_definition_simplekeys() {
+ $instance = cache_config_phpunittest::instance();
+ $instance->phpunit_add_definition('phpunit/simplekeytest', array(
+ 'mode' => cache_store::MODE_APPLICATION,
+ 'component' => 'phpunit',
+ 'area' => 'simplekeytest',
+ 'simplekeys' => true
+ ));
+ $cache = cache::make('phpunit', 'simplekeytest');
+
+ $this->assertTrue($cache->set('testkey1', 'test data 1'));
+ $this->assertEquals('test data 1', $cache->get('testkey1'));
+ $this->assertTrue($cache->set('testkey2', 'test data 2'));
+ $this->assertEquals('test data 2', $cache->get('testkey2'));
+
+ $cache->purge();
+
+ $this->assertTrue($cache->set('1', 'test data 1'));
+ $this->assertEquals('test data 1', $cache->get('1'));
+ $this->assertTrue($cache->set('2', 'test data 2'));
+ $this->assertEquals('test data 2', $cache->get('2'));
+ }
+
public function test_definition_ttl() {
$instance = cache_config_phpunittest::instance(true);
$instance->phpunit_add_definition('phpunit/ttltest', array(
@@ -515,13 +559,15 @@ class cache_phpunit_tests extends advanced_testcase {
// OK data added, data invalidated, and invalidation time has been set.
// Now we need to manually add back the data and adjust the invalidation time.
- $timefile = $CFG->dataroot.'/cache/cachestore_file/default_application/phpunit_eventinvalidationtest/494515064.cache';
+ $timefile = $CFG->dataroot.'/cache/cachestore_file/default_application/phpunit_eventinvalidationtest/a65/a65b1dc524cf6e03c1795197c84d5231eb229b86.cache';
$timecont = serialize(cache::now() - 60); // Back 60sec in the past to force it to re-invalidate.
+ make_writable_directory(dirname($timefile));
file_put_contents($timefile, $timecont);
$this->assertTrue(file_exists($timefile));
- $datafile = $CFG->dataroot.'/cache/cachestore_file/default_application/phpunit_eventinvalidationtest/3140056538.cache';
+ $datafile = $CFG->dataroot.'/cache/cachestore_file/default_application/phpunit_eventinvalidationtest/626/626e9c7a45febd98f064c2b383de8d9d4ebbde7b.cache';
$datacont = serialize("test data 1");
+ make_writable_directory(dirname($datafile));
file_put_contents($datafile, $datacont);
$this->assertTrue(file_exists($datafile));
diff --git a/lib/db/caches.php b/lib/db/caches.php
index acf9b0eab60..5d44fd1241e 100644
--- a/lib/db/caches.php
+++ b/lib/db/caches.php
@@ -29,14 +29,19 @@
$definitions = array(
// Used to store processed lang files.
+ // The keys used are the component of the string file.
'string' => array(
'mode' => cache_store::MODE_APPLICATION,
+ 'simplekeys' => true,
'simpledata' => true,
'persistent' => true,
'persistentmaxsize' => 3
),
// Used to store database meta information.
+ // The database meta information includes information about tables and there columns.
+ // Its keys are the table names.
+ // When creating an instance of this definition you must provide the database family that is being used.
'databasemeta' => array(
'mode' => cache_store::MODE_APPLICATION,
'requireidentifiers' => array(
@@ -47,13 +52,24 @@ $definitions = array(
),
// Used to store data from the config + config_plugins table in the database.
+ // The key used is the component:
+ // - core for all core config settings
+ // - plugin component for all plugin settings.
+ // Persistence is used because normally several settings within a script.
'config' => array(
'mode' => cache_store::MODE_APPLICATION,
'persistent' => true,
+ 'simplekeys' => true,
'simpledata' => true
),
// Event invalidation cache.
+ // This cache is used to manage event invalidation, its keys are the event names.
+ // Whenever something is invalidated it is both purged immediately and an event record created with the timestamp.
+ // When a new cache is initialised all timestamps are looked at and if past data is once more invalidated.
+ // Data guarantee is required in order to ensure invalidation always occurs.
+ // Persistence has been turned on as normally events are used for frequently used caches and this event invalidation
+ // cache will likely be used either lots or never.
'eventinvalidation' => array(
'mode' => cache_store::MODE_APPLICATION,
'persistent' => true,
@@ -66,6 +82,7 @@ $definitions = array(
// question_bank::load_question.
'questiondata' => array(
'mode' => cache_store::MODE_APPLICATION,
+ 'simplekeys' => true, // The id of the question is used.
'requiredataguarantee' => false,
'datasource' => 'question_finder',
'datasourcefile' => 'question/engine/bank.php',