From 687ea08633a7286de3b8bc98a530ae57bdb5f67b Mon Sep 17 00:00:00 2001 From: Ryan Cramer Date: Fri, 31 Jan 2020 11:25:08 -0500 Subject: [PATCH] Add new FieldsTableTools class as a helper for managing field tables and their indexes. Intended for non-public API (internal core) use, but methods can be accessed $fields->tableTools() --- wire/core/Field.php | 26 ++- wire/core/Fields.php | 23 +- wire/core/FieldsTableTools.php | 402 +++++++++++++++++++++++++++++++++ wire/core/Fieldtype.php | 15 +- 4 files changed, 462 insertions(+), 4 deletions(-) create mode 100644 wire/core/FieldsTableTools.php diff --git a/wire/core/Field.php b/wire/core/Field.php index eac9e54d..cbc2a95a 100644 --- a/wire/core/Field.php +++ b/wire/core/Field.php @@ -35,6 +35,7 @@ * @property array|null $orderByCols Columns that WireArray values are sorted by (default=null), Example: "sort" or "-created". #pw-internal * @property int|null $paginationLimit Used by paginated WireArray values to indicate limit to use during load. #pw-internal * @property array $allowContexts Names of settings that are custom configured to be allowed for context. #pw-group-properties + * @property bool|int|null $flagUnique Non-empty value indicates request for, or presence of, Field::flagUnique flag. #pw-internal * * Common Inputfield properties that Field objects store: * @property int|bool|null $required Whether or not this field is required during input #pw-group-properties @@ -56,6 +57,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Field should be automatically joined to the page at page load time + * * #pw-group-flags * */ @@ -63,6 +65,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Field used by all fieldgroups - all fieldgroups required to contain this field + * * #pw-group-flags * */ @@ -70,6 +73,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Field is a system field and may not be deleted, have it's name changed, or be converted to non-system + * * #pw-group-flags * */ @@ -77,6 +81,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Field is permanent in any fieldgroups/templates where it exists - it may not be removed from them + * * #pw-group-flags * */ @@ -84,6 +89,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Field is access controlled + * * #pw-group-flags * */ @@ -93,6 +99,7 @@ class Field extends WireData implements Saveable, Exportable { * If field is access controlled, this flag says that values are still front-end API accessible * * Without this flag, non-viewable values are made blank when output formatting is ON. + * * #pw-group-flags * */ @@ -102,13 +109,28 @@ class Field extends WireData implements Saveable, Exportable { * If field is access controlled and user has no edit access, they can still view in the editor (if they have view permission) * * Without this flag, non-editable values are simply not shown in the editor at all. + * * #pw-group-flags * */ - const flagAccessEditor = 128; + const flagAccessEditor = 128; + + /** + * Field requires that the same value is not repeated more than once in its table 'data' column (when supported by Fieldtype) + * + * When this flag is set and there is a non-empty $flagUnique property on the field, then it indicates a unique index + * is currently present. When only this flag is present (no property), it indicates a request to remove the index and flag. + * When only the property is present (no flag), it indicates a pending request to add unique index and flag. + * + * #pw-group-flags + * @since 3.0.150 + * + */ + const flagUnique = 256; /** * Field has been placed in a runtime state where it is contextual to a specific fieldgroup and is no longer saveable + * * #pw-group-flags * */ @@ -116,6 +138,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Set this flag to override system/permanent flags if necessary - once set, system/permanent flags can be removed, but not in the same set(). + * * #pw-group-flags * */ @@ -123,6 +146,7 @@ class Field extends WireData implements Saveable, Exportable { /** * Prefix for database tables + * * #pw-internal * */ diff --git a/wire/core/Fields.php b/wire/core/Fields.php index 0301ef88..05a43a4d 100644 --- a/wire/core/Fields.php +++ b/wire/core/Fields.php @@ -113,6 +113,12 @@ class Fields extends WireSaveableItems { */ protected $tagList = null; + /** + * @var FieldsTableTools|null + * + */ + protected $tableTools = null; + /** * Construct * @@ -181,7 +187,8 @@ class Fields extends WireSaveableItems { if(strpos($class, "\\") === false) $class = wireClassName($class, true); if(!class_exists($class)) return parent::makeItem($a); - + + /** @var Field $field */ $field = new $class(); $this->wire($field); @@ -1113,5 +1120,19 @@ class Fields extends WireSaveableItems { return $fieldtypes; } + /** + * Get FieldsIndexTools instance + * + * #pw-internal + * + * @return FieldsTableTools + * @since 3.0.150 + * + */ + public function tableTools() { + if($this->tableTools === null) $this->tableTools = $this->wire(new FieldsTableTools()); + return $this->tableTools; + } + } diff --git a/wire/core/FieldsTableTools.php b/wire/core/FieldsTableTools.php new file mode 100644 index 00000000..e2002e6d --- /dev/null +++ b/wire/core/FieldsTableTools.php @@ -0,0 +1,402 @@ +tableTools()`. + * + * ProcessWire 3.x, Copyright 2020 by Ryan Cramer + * https://processwire.com + * + * @since 3.0.150 + * + * #pw-internal + * + */ +class FieldsTableTools extends Wire { + + /** + * Find duplicate rows for a specific column in a field’s table + * + * #pw-internal + * + * @param Field $field + * @param array $options + * - `column` (string): Name of column to find duplicate values in (default='data') + * - `value` (bool|string): Value to find duplicates of, or false to find all duplicate values (default=false) + * - `verbose` (bool): Include entire DB rows in returned result? (default=false) + * @return array Returns array of arrays where each item contains indexes of 'count' (int) and 'value' (int|string), plus, + * if the `verbose` option is true, returned value also adds a `rows` index (array) containing contents of entire matching DB rows. + * + */ + public function findDuplicateRows(Field $field, array $options = array()) { + + $defaults = array( + 'column' => 'data', + 'value' => false, + 'verbose' => false, + ); + + $options = array_merge($defaults, $options); + $result = array(); + + /** @var WireDatabasePDO $database */ + $database = $this->wire('database'); + $table = $database->escapeTable($field->getTable()); + $col = $database->escapeCol($options['column']); + $sql = "SELECT $col, COUNT($col) FROM $table "; + + if($options['value'] !== false) { + if($options['value'] === null) { + $sql .= "WHERE $col IS NULL "; + } else { + $sql .= "WHERE $col=:val "; + } + } + + $sql .= "GROUP BY $col HAVING COUNT($col) > 1"; + $query = $database->prepare($sql); + + if($options['value'] !== false && $options['value'] !== null) { + $query->bindValue(':val', $options['value']); + } + + $query->execute(); + + while($row = $query->fetch(\PDO::FETCH_NUM)) { + $result[] = array('value' => $row[0], 'count' => (int) $row[1]); + } + + $query->closeCursor(); + + if($options['verbose']) { + foreach($result as $key => $item) { + $result[$key]['rows'] = array(); + $sql = "SELECT * FROM $table WHERE $col=:val"; + $query = $database->prepare($sql); + $query->bindValue(':val', $item['value']); + $query->execute(); + while($row = $query->fetch(\PDO::FETCH_ASSOC)) { + $result[$key]['rows'][] = $row; + } + $query->closeCursor(); + } + } + + return $result; + } + + /** + * Add or remove a unique index for a field on its 'data' column + * + * #pw-internal + * + * @param Field $field + * @param bool $add Specify false to remove index rather than add (default=true) + * @return bool|int Returns one of the following when adding index: + * - `true` (bool): When index successfully added. + * - `false` (bool): Index cannot be added because there are non-unique rows already present (not allowed). + * - `1` (int): Unique index already present so was not necessary (not needed). + * - `0` (int): Requested column does not exist in table so cannot be added as index (not allowed). + * Returns one of the following when removing index: + * - `true` (bool): When index successfully removed. + * - `false` (bool): When index failed to remove. + * - `1` (int): When remove index but there is no unique index to remove (not needed). + * - `0` (int): When remove index that is not one we have previously added (not allowed). + * @throws \PDOException When given invalid column name or unknown error condition + * + */ + public function setUniqueIndex(Field $field, $add = true) { + + /** @var WireDatabasePDO $database */ + $database = $this->wire('database'); + $col = 'data'; + $table = $database->escapeTable($field->getTable()); + $uniqueIndexName = $this->hasUniqueIndex($field, $col); + $requireIndexName = $database->escapeCol($col . '_unique'); + $action = ''; // whether to 'add' or 'remove' flag and property from Field + + if($uniqueIndexName) { + // already has unique index for indicated column + if($add) { + // already has unique index name + $result = 1; + $action = 'add'; + + } else { + // remove requested + if($uniqueIndexName === $requireIndexName) { + // remove the unique index + try { + $result = $database->exec("ALTER TABLE $table DROP INDEX `$requireIndexName`"); + if($result) $action = 'remove'; + } catch(\Exception $e) { + $result = false; + } + } else { + // unique index present but it’s not one we previously added + $result = 0; + $action = 'remove'; + } + } + + } else if($add) { + // no unique index yet exists for column, so add one + $col = $database->escapeCol($col); + $sql = "ALTER TABLE $table ADD UNIQUE `$requireIndexName` (`$col`)"; + try { + $result = $database->exec($sql); + if($result) $action = 'add'; + } catch(\Exception $e) { + $action = 'remove'; + if($e->getCode() == 23000) { + // non unique rows already present + $result = false; + } else if($e->getCode() == 42000) { + // requested column does not exist + $result = 0; + } else { + throw $e; + } + } + + } else { + // remove properties indicating unique + if($field->hasFlag(Field::flagUnique) || $field->flagUnique) $action = 'remove'; + $result = 1; + } + + if($action) { + $save = false; + if($action === 'add') { + if(!$field->hasFlag(Field::flagUnique)) $save = $field->addFlag(Field::flagUnique); + if(!$field->flagUnique) $save = $field->set('flagUnique', true); + } else if($action === 'remove') { + if($field->hasFlag(Field::flagUnique)) $save = $field->removeFlag(Field::flagUnique); + if($field->flagUnique) $save = $field->remove('flagUnique'); + } + if($save) $field->save(); + } + + return $result; + } + + /** + * Does given field have a unique index on column? + * + * #pw-internal + * + * @param Field $field + * @param string $col + * @return bool|string Returns index name when present, or boolean false when not + * + */ + public function hasUniqueIndex(Field $field, $col = 'data') { + /** @var WireDatabasePDO $database */ + $database = $this->wire('database'); + $table = $database->escapeTable($field->getTable()); + $sql = "SHOW INDEX FROM $table"; + $query = $database->prepare($sql); + $query->execute(); + $has = false; + while($row = $query->fetch(\PDO::FETCH_ASSOC)) { + if($row['Column_name'] === $col && !$row['Non_unique']) { + $has = $row['Key_name']; + break; + } + } + $query->closeCursor(); + return $has; + } + + /** + * Check state of field unique 'data' index and update as needed + * + * @param Field $field + * @param bool $verbose Show messages when changes made? (default=true) + * @throws WireException + * + */ + public function checkUniqueIndex(Field $field, $verbose = true) { + + static $checking = false; + if($checking) return; + + $col = 'data'; + $session = $this->wire('session'); + if($verbose && !$session) return; + + // is unique index requested? + $useUnique = (bool) $field->get('flagUnique'); + + // ise unique index already present? + $hasUnique = (bool) $field->hasFlag(Field::flagUnique); + + if($useUnique === $hasUnique) return; + + if(!$this->database->tableExists($field->getTable())) return; + + $checking = true; + + if($useUnique && !$hasUnique) { + // add unique index + $qty = $this->deleteEmptyRows($field, $col); + + if($qty && $verbose) { + $session->message(sprintf($this->_('Deleted %d empty row(s) for field %s'), $qty, $field->name)); + } + + $result = $this->setUniqueIndex($field, true); + + if($result === false && $verbose) { + $msg = $this->_('Unique index cannot be added yet because there are already non-unique row(s) present:') . ' '; + $rows = $this->findDuplicateRows($field, array('verbose' => true, 'column' => $col)); + foreach($rows as $row) { + $ids = array(); + foreach($row['rows'] as $a) { + $ids[] = $a['pages_id']; + } + $msg .= "\n• $row[value] — " . + sprintf($this->_('Appears %d times'), $row['count']) . ' ' . + sprintf($this->_('(pages: %s)'), implode(', ', $ids)) . ' '; + } + $session->error($msg, Notice::noGroup); + + } else if($result && $verbose) { + $session->message($this->_('Added unique index')); + } + + } else if($hasUnique && !$useUnique) { + // remove unique index + $result = $this->setUniqueIndex($field, false); + if($result && $verbose) $session->message($this->_('Removed unique index')); + } + + $checking = false; + } + + /** + * Delete rows having empty column value + * + * @param Field $field + * @param string $col Column name (default='data') + * @param bool $strict When true, delete not allowed if there are columns other than one given and 'pages_id' (default=true) + * @return bool|int Returns false if delete not allowed, otherwise returns int with # of rows deleted + * @throws WireException + * + */ + public function deleteEmptyRows(Field $field, $col = 'data', $strict = true) { + + /** @var WireDatabasePDO $database */ + $database = $this->wire('database'); + $table = $database->escapeTable($field->getTable()); + $fieldtype = $field->type; + $schema = $fieldtype->getDatabaseSchema($field); + $wheres = array(); + + $types = array( + 'INT', 'TINYINT', 'SMALLINT', 'MEDIUMINT', 'BIGINT', + 'TEXT', 'TINYTEXT', 'MEDIUMTEXT', 'LONGTEXT', + 'DATE', 'TIME', 'DATETIME', 'TIMESTAMP', + 'CHAR', 'VARCHAR', + ); + + unset($schema['keys'], $schema['pages_id'], $schema['xtra']); + + if(!isset($schema[$col])) return false; // if there's no schema for this column, fail + if($strict && count($schema) > 1) return false; // if there are other columns too, fail + + $type = strtoupper($schema[$col]); + $allowNull = strpos($type, 'NOT NULL') === false; + + if(strpos($type, ' ')) list($type,) = explode(' ', $type, 2); + if(strpos($type, '(')) list($type,) = explode('(', $type, 2); + + if(!in_array(trim($type), $types)) return false; // if not in allowed col types, fail + + if($col !== 'data') { + $col = $database->escapeCol($this->sanitizer->fieldName($col)); + if(empty($col)) return false; + } + + if(strpos($type, 'INT') !== false) { + if($fieldtype->isEmptyValue($field, 0)) { + $wheres[] = "$col=0"; + } + } else if($fieldtype->isEmptyValue($field, '')) { + $wheres[] = "$col=''"; + } + + if($allowNull) { + $wheres[] = "$col IS NULL"; + } + + if(count($wheres)) { + // delete empty rows matching our conditions + $sql = "DELETE FROM $table WHERE " . implode(' OR ', $wheres); + $query = $database->prepare($sql); + $result = $query->execute() ? $query->rowCount() : 0; + $query->closeCursor(); + } else { + // no empty rows possible + $result = true; + } + + return $result; + } + + /** + * Create a checkbox Inputfield to configure unique value state + * + * @param Field $field + * @return InputfieldCheckbox + * + */ + public function getUniqueIndexInputfield(Field $field) { + + $col = 'data'; + $modules = $this->wire('modules'); /** @var Modules $modules */ + + if((bool) $field->flagUnique != $field->hasFlag(Field::flagUnique)) { + $this->checkUniqueIndex($field, true); + } + + $f = $modules->get('InputfieldCheckbox'); /** @var InputfieldCheckbox $f */ + $f->attr('name', "flagUnique"); + $f->label = $this->_('Unique'); + $f->icon = 'hand-stop-o'; + $f->description = $this->_('When checked, a given value may not be used more than once in this field, and thus may not appear on more than one page.'); + + if($this->hasUniqueIndex($field, $col)) { + $f->attr('checked', 'checked'); + if(!$field->hasFlag(Field::flagUnique)) $field->addFlag(Field::flagUnique); + if(!$field->flagUnique) $field->flagUnique = true; + } + + return $f; + } + + /** + * Does given value exist anywhere in field table? + * + * @param Field $field + * @param string|int $value + * @param string $col + * @return int Returns page ID where value exists, if found. Otherwise returns 0. + * @throws WireException + * + */ + public function valueExists(Field $field, $value, $col = 'data') { + /** @var WireDatabasePDO $database */ + $database = $this->wire('database'); + $table = $database->escapeTable($field->getTable()); + if($col !== 'data') $col = $database->escapeCol($this->sanitizer->fieldName($col)); + $sql = "SELECT pages_id FROM $table WHERE $col=:val LIMIT 1"; + $query = $database->prepare($sql); + $query->bindValue(':val', $value); + $query->execute(); + $pageId = $query->rowCount() ? (int) $query->fetchColumn() : 0; + $query->closeCursor(); + return $pageId; + } + +} \ No newline at end of file diff --git a/wire/core/Fieldtype.php b/wire/core/Fieldtype.php index b1b5a228..c0f09a17 100644 --- a/wire/core/Fieldtype.php +++ b/wire/core/Fieldtype.php @@ -1135,7 +1135,7 @@ abstract class Fieldtype extends WireData implements Module { * @param Page $page Page object to save. * @param Field $field Field to retrieve from the page. * @return bool True on success, false on DB save failure. - * @throws WireException + * @throws WireException|\PDOException|WireDatabaseException * */ public function ___savePageField(Page $page, Field $field) { @@ -1200,7 +1200,18 @@ abstract class Fieldtype extends WireData implements Module { } $query = $database->prepare($sql); - $result = $query->execute(); + + try { + $result = $query->execute(); + + } catch(\PDOException $e) { + if($e->getCode() == 23000) { + $message = sprintf($this->_('Value not allowed for field “%2$s” because it is already in use'), $field->name); + throw new WireDatabaseException($message, $e->getCode(), $e); + } else { + throw $e; + } + } return $result; }