diff --git a/wire/core/DatabaseQuerySelectFulltext.php b/wire/core/DatabaseQuerySelectFulltext.php index 4a1d319f..badaa846 100644 --- a/wire/core/DatabaseQuerySelectFulltext.php +++ b/wire/core/DatabaseQuerySelectFulltext.php @@ -37,7 +37,7 @@ class DatabaseQuerySelectFulltext extends Wire { const maxQueryValueLength = 500; /** - * @var DatabaseQuerySelect + * @var DatabaseQuerySelect|PageFinderDatabaseQuerySelect * */ protected $query; @@ -67,7 +67,9 @@ class DatabaseQuerySelectFulltext extends Wire { protected $method = ''; /** - * Is it a NOT operator? + * Is it a NOT operator? + * + * This is not used by PageFinder originating queries, which handles NOT internally. * * @var bool * @@ -82,6 +84,22 @@ class DatabaseQuerySelectFulltext extends Wire { */ protected $minWordLength = null; + /** + * Allow adding 'ORDER BY' to query? + * + * @var bool|null + * + */ + protected $allowOrder = null; + + /** + * Allow fulltext searches to fallback to LIKE searches to match stopwords? + * + * @var bool + * + */ + protected $allowStopwords = true; + /** * @var array * @@ -110,10 +128,11 @@ class DatabaseQuerySelectFulltext extends Wire { /** * Construct * - * @param DatabaseQuerySelect $query + * @param DatabaseQuerySelect|PageFinderDatabaseQuerySelect $query * */ public function __construct(DatabaseQuerySelect $query) { + $query->wire($this); $this->query = $query; } @@ -148,6 +167,40 @@ class DatabaseQuerySelectFulltext extends Wire { return "$this->tableName.$this->fieldName"; } + /** + * Get or set whether or not 'ORDER BY' statements are allowed to be added + * + * @param null|bool $allow Specify bool to set or omit to get + * @return bool|null Returns bool when known or null when not yet known + * @since 3.0.162 + * + */ + public function allowOrder($allow = null) { + if($allow !== null) $this->allowOrder = $allow ? true : false; + return $this->allowOrder; + } + + /** + * Get or set whether fulltext searches can fallback to LIKE searches to match stopwords + * + * @param null|bool $allow Specify bool to set or omit to get + * @return bool + * @since 3.0.162 + * + */ + public function allowStopwords($allow = null) { + if($allow !== null) $this->allowStopwords = $allow ? true : false; + return $this->allowStopwords; + } + + /** + * @return string + * + */ + protected function matchType() { + return "\n " . ($this->not ? 'NOT MATCH' : 'MATCH'); + } + /** * Escape string for use in a MySQL LIKE * @@ -204,11 +257,19 @@ class DatabaseQuerySelectFulltext extends Wire { $this->tableName = $this->database->escapeTable($tableName); $this->fieldName = $this->database->escapeCol($fieldName); + $allowOrder = true; if(strpos($operator, '!') === 0 && $operator !== '!=') { $this->not = true; $operator = ltrim($operator, '!'); + } else { + // disable orderby statements when calling object will be negating whatever we do + $selector = $this->query->selector; + if($selector && $selector instanceof Selector && $selector->not) $allowOrder = false; } + + // if allowOrder has not been specifically set, then set value now + if($this->allowOrder === null) $this->allowOrder = $allowOrder; $this->operator = $operator; @@ -275,7 +336,8 @@ class DatabaseQuerySelectFulltext extends Wire { * */ protected function matchEquals($value) { - $this->query->where("$this->tableField$this->operator?", $value); + $op = $this->wire()->database->escapeOperator($this->operator, WireDatabasePDO::operatorTypeComparison); + $this->query->where("$this->tableField$op?", $value); } /** @@ -367,9 +429,10 @@ class DatabaseQuerySelectFulltext extends Wire { $partial = strpos($operator, '*') !== false; $partialLast = $operator === '~~='; $expand = strpos($operator, '+') !== false; - $matchType = $this->not ? 'NOT MATCH' : 'MATCH'; + $matchType = $this->matchType(); $scoreField = $this->getScoreFieldName(); $matchAgainst = ''; + $wheres = array(); $data = $this->getBooleanModeWords($value, array( 'required' => $required, @@ -378,18 +441,27 @@ class DatabaseQuerySelectFulltext extends Wire { 'partialLess' => ($partial || $expand), 'alternates' => $expand, )); - + + if(empty($data['value'])) { + // query contains no indexable words: force non-match + //$this->query->where('1>2'); + //return; + // TEST OUT: title|summary~|+=beer + } + if($expand) { - if(!empty($data['booleanValue'])) { + if(!empty($data['booleanValue']) && $this->allowOrder) { // ensure full matches are above expanded matches $preScoreField = $this->getScoreFieldName(); $bindKey = $this->query->bindValueGetKey($data['booleanValue']); $this->query->select("$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE) + 111.1 AS $preScoreField"); $this->query->orderby("$preScoreField DESC"); } - $bindValue = trim($data['value'] . ' ' . implode(' ', $data['altWords'])); - $bindKey = $this->query->bindValueGetKey($this->escapeAgainst($bindValue)); - $matchAgainst = "$matchType($tableField) AGAINST($bindKey WITH QUERY EXPANSION)"; + if(!empty($data['matchValue'])) { + $bindValue = trim($data['matchValue']); + $bindKey = $this->query->bindValueGetKey($this->escapeAgainst($bindValue)); + $matchAgainst = "$matchType($tableField) AGAINST($bindKey WITH QUERY EXPANSION)"; + } } else if(!empty($data['booleanValue'])) { $bindKey = $this->query->bindValueGetKey($data['booleanValue']); @@ -397,16 +469,26 @@ class DatabaseQuerySelectFulltext extends Wire { } if($matchAgainst) { - $this->query->where($matchAgainst); - $this->query->select("$matchAgainst AS $scoreField"); - $this->query->orderby("$scoreField DESC"); + $wheres[] = $matchAgainst; + // $this->query->where($matchAgainst); + if($this->allowOrder) { + $this->query->select("$matchAgainst AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } + } else if(!$this->allowStopwords) { + // no match possible + // $this->query->where('1>2'); + $wheres[] = '1>2'; } if(!empty($data['likeWords'])) { // stopwords or words that were too short to use fulltext index - $wheres = array(); $likeType = $this->not ? 'NOT RLIKE' : 'RLIKE'; + $orLikes = array(); + $andLikes = array(); foreach($data['likeWords'] as $word) { + $isStopword = isset($data['stopWords'][$word]); + if($isStopword && !$this->allowStopwords) continue; $word = $this->escapeLike($word); if(!strlen($word)) continue; $likeValue = '([[:blank:]]|[[:punct:]]|[[:space:]]|>|^)' . preg_quote($word); @@ -416,13 +498,28 @@ class DatabaseQuerySelectFulltext extends Wire { // match to word-end $likeValue .= '([[:blank:]]|[[:punct:]]|[[:space:]]|<|$)'; } - $bindKey = $this->query->bindValueGetKey($likeValue); - $wheres[] = "($tableField $likeType $bindKey)"; + $bindKey = $this->query->bindValueGetKey($likeValue); + $likeWhere = "($tableField $likeType $bindKey)"; + if(!$required || ($isStopword && $expand)) { + $orLikes[] = $likeWhere; + } else { + $andLikes[] = $likeWhere; + } } - if(count($wheres)) { - $and = $required ? ' AND ' : ' OR '; - $this->query->where(implode($and, $wheres)); + $whereLike = ''; + if(count($orLikes)) { + $whereLike .= '(' . implode(' OR ', $orLikes) . ')'; + if(count($andLikes)) $whereLike .= $required ? ' AND ' : ' OR '; } + if(count($andLikes)) { + $whereLike .= implode(' AND ', $andLikes); + } + if($whereLike) $wheres[] = $whereLike; + } + + if(count($wheres)) { + $and = $required ? ' AND ' : ' OR '; + $this->query->where('(' . implode($and, $wheres) . ')'); } } @@ -435,7 +532,6 @@ class DatabaseQuerySelectFulltext extends Wire { protected function matchPhrase($value) { $tableField = $this->tableField(); - $not = strpos($this->operator, '!') === 0; $likeValue = ''; $words = $this->words($value); $lastWord = count($words) > 1 ? array_pop($words) : ''; @@ -460,10 +556,10 @@ class DatabaseQuerySelectFulltext extends Wire { if($lastWord !== '' || !strlen($againstValue)) { // match entire phrase with LIKE as secondary qualifier that includes last word // so that we can perform a partial match on the last word only. This is necessary - // because we can’t use partial match qualifiers in or out of quoted phrases - // if word is indexable let it contribute to final score + // because we can’t use partial match qualifiers in or out of quoted phrases. $lastWord = strlen($lastWord) ? $this->escapeAgainst($lastWord) : ''; if(strlen($lastWord) && $this->isIndexableWord($lastWord)) { + // if word is indexable let it contribute to final score // expand the againstValue to include the last word as a required partial match $againstValue = trim("$againstValue +$lastWord*"); } @@ -473,19 +569,22 @@ class DatabaseQuerySelectFulltext extends Wire { if(strlen($againstValue)) { // use MATCH/AGAINST $bindKey = $this->query->bindValueGetKey($againstValue); - $match = $not ? 'NOT MATCH' : 'MATCH'; - $matchAgainst = "$match($tableField) AGAINST($bindKey IN BOOLEAN MODE)"; - $scoreField = $this->getScoreFieldName(); - $this->query->select("$matchAgainst AS $scoreField"); + $matchType = $this->matchType(); + $matchAgainst = "$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE)"; $this->query->where($matchAgainst); - $this->query->orderby("$scoreField DESC"); + + if($this->allowOrder) { + $scoreField = $this->getScoreFieldName(); + $this->query->select("$matchAgainst AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } } if(strlen($likeValue)) { // LIKE is used as a secondary qualifier to MATCH/AGAINST so that it is // performed only on rows already identified from FULLTEXT index, unless // no MATCH/AGAINST could be created due to stopwords or too-short words - $likeType = $not ? 'NOT RLIKE' : 'RLIKE'; + $likeType = $this->not ? 'NOT RLIKE' : 'RLIKE'; $this->query->where("($tableField $likeType ?)", $likeValue); } } @@ -499,8 +598,7 @@ class DatabaseQuerySelectFulltext extends Wire { protected function matchPhraseExpand($value) { $tableField = $this->tableField(); - $not = strpos($this->operator, '!') === 0; - $matchType = $not ? "\nNOT MATCH" : "\nMATCH"; + $matchType = $this->matchType(); $words = $this->words($value, array('indexable' => true)); $wordsAlternates = array(); @@ -509,8 +607,11 @@ class DatabaseQuerySelectFulltext extends Wire { $againstValue = '+"' . $this->escapeAgainst($value) . '*"'; $bindKey = $this->query->bindValueGetKey($againstValue); $matchAgainst = "$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE)"; - $this->query->select("$matchAgainst + 333.3 AS $scoreField"); - $this->query->orderby("$scoreField DESC"); + + if($this->allowOrder) { + $this->query->select("$matchAgainst + 333.3 AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } if(!count($words)) { // no words to work with for query expansion (not likely, unless stopwords or too-short) @@ -542,19 +643,26 @@ class DatabaseQuerySelectFulltext extends Wire { } $againstValue .= ") "; } - $bindKey = $this->query->bindValueGetKey(trim($againstValue)); - $this->query->select("$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE) + 222.2 AS $scoreField"); - $this->query->orderby("$scoreField DESC"); + + if($this->allowOrder && strlen($againstValue)) { + $bindKey = $this->query->bindValueGetKey(trim($againstValue)); + $this->query->select("$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE) + 222.2 AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } // QUERY EXPANSION: regular match/against words with query expansion $words = array_unique(array_merge($words, $wordsAlternates)); $againstValue = $this->escapeAgainst(implode(' ', $words)); $bindKey = $this->query->bindValueGetKey($againstValue); $matchAgainst = "$matchType($tableField) AGAINST($bindKey WITH QUERY EXPANSION)"; - $scoreField = $this->getScoreFieldName(); $this->query->where($matchAgainst); + + $scoreField = $this->getScoreFieldName(); $this->query->select("$matchAgainst AS $scoreField"); - $this->query->orderby("$scoreField DESC"); + + if($this->allowOrder) { + $this->query->orderby("$scoreField DESC"); + } } /** @@ -570,9 +678,9 @@ class DatabaseQuerySelectFulltext extends Wire { $tableField = $this->tableField(); $expand = strpos($this->operator, '+') !== false; - $matchType = $this->not ? 'NOT MATCH' : 'MATCH'; + $matchType = $this->matchType(); - if($expand) { + if($expand && $this->allowOrder) { // boolean mode query for sorting purposes $scoreField = $this->getScoreFieldName(); $data = $this->getBooleanModeWords($value, array( @@ -595,7 +703,7 @@ class DatabaseQuerySelectFulltext extends Wire { $againstValue = $this->escapeAgainst(implode(' ', $words)); if(!count($words) || !strlen(trim($againstValue))) { - // query contains no indexbale words: force non-match + // query contains no indexable words: force non-match if(strlen($value)) $this->query->where('1>2'); return; } @@ -603,9 +711,11 @@ class DatabaseQuerySelectFulltext extends Wire { $bindKey = $this->query->bindValueGetKey($againstValue); $againstType = $expand ? 'WITH QUERY EXPANSION' : ''; $where = "$matchType($tableField) AGAINST($bindKey $againstType)"; - $this->query->select("$where AS $scoreField"); - $this->query->where($where); - $this->query->orderby("$scoreField DESC"); + $this->query->where($where); + if($this->allowOrder) { + $this->query->select("$where AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } } /** @@ -623,7 +733,6 @@ class DatabaseQuerySelectFulltext extends Wire { // $= Ends with $tableField = $this->tableField(); - $not = strpos($this->operator, '!') === 0; $matchStart = strpos($this->operator, '^') !== false; $againstValue = ''; @@ -643,15 +752,17 @@ class DatabaseQuerySelectFulltext extends Wire { if(strlen($againstValue)) { // use MATCH/AGAINST to pre-filter before RLIKE when possible $bindKey = $this->query->bindValueGetKey($againstValue); - $match = $not ? 'NOT MATCH' : 'MATCH'; - $matchAgainst = "$match($tableField) AGAINST($bindKey IN BOOLEAN MODE)"; + $matchType = $this->matchType(); + $matchAgainst = "$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE)"; $scoreField = $this->getScoreFieldName(); - $this->query->select("$matchAgainst AS $scoreField"); $this->query->where($matchAgainst); - $this->query->orderby("$scoreField DESC"); + if($this->allowOrder) { + $this->query->select("$matchAgainst AS $scoreField"); + $this->query->orderby("$scoreField DESC"); + } } - $likeType = $not ? 'NOT RLIKE' : 'RLIKE'; + $likeType = $this->not ? 'NOT RLIKE' : 'RLIKE'; $likeValue = preg_quote($value); if($matchStart) { @@ -677,11 +788,14 @@ class DatabaseQuerySelectFulltext extends Wire { $scoreField = $this->getScoreFieldName(); $against = $this->getBooleanModeCommands($text); $bindKey = $this->query->bindValueGetKey($against); - $matchAgainst = "MATCH($tableField) AGAINST($bindKey IN BOOLEAN MODE) "; - $select = "$matchAgainst AS $scoreField "; - $this->query->select($select); - $this->query->orderby("$scoreField DESC"); + $matchType = $this->matchType(); + $matchAgainst = "$matchType($tableField) AGAINST($bindKey IN BOOLEAN MODE) "; $this->query->where($matchAgainst); + if($this->allowOrder) { + $select = "$matchAgainst AS $scoreField "; + $this->query->select($select); + $this->query->orderby("$scoreField DESC"); + } } /** @@ -745,7 +859,7 @@ class DatabaseQuerySelectFulltext extends Wire { if($this->isStopword($word)) { // handle stop-word $stopWords[$word] = $word; - if($useStopwords) $booleanValues[$word] = "$word*"; + if($useStopwords && $partial) $booleanValues[$word] = "<$word*"; continue; // do nothing further with stopwords } else if($length < $minWordLength) { @@ -756,7 +870,7 @@ class DatabaseQuerySelectFulltext extends Wire { } else if($options['partialLess']) { // handle regular word and match full word (more weight), or partial word (less weight) - $booleanValues[$word] = $required . "(>$word $word*)"; + $booleanValues[$word] = $required ? "+(>$word $word*)" : "$word*"; $goodWords[$word] = $word; } else { @@ -771,6 +885,7 @@ class DatabaseQuerySelectFulltext extends Wire { if($booleanValue !== $booleanValues[$word]) { $booleanValues[$word] = $booleanValue; $altWords = array_merge($altWords, $alternates); + $allWords = array_merge($allWords, $altWords); } } } @@ -780,6 +895,16 @@ class DatabaseQuerySelectFulltext extends Wire { $lastRequired = isset($stopWords[$lastWord]) ? '' : $required; $booleanValues[$lastWord] = $lastRequired . $lastWord . '*'; } + + if($useStopwords && !$required && count($stopWords) && count($goodWords)) { + // increase weight of non-stopwords + foreach($goodWords as $word) { + $booleanWord = $booleanValues[$word]; + if(!in_array($booleanWord[0], array('(', '+', '<', '>', '-', '~', '"'))) { + $booleanValues[$word] = ">$booleanWord"; + } + } + } $badWords = array_merge($stopWords, $shortWords); @@ -787,7 +912,7 @@ class DatabaseQuerySelectFulltext extends Wire { $numOkayWords = count($goodWords) + count($shortWords); foreach($stopWords as $word) { $likeWords[$word] = $word; - if($numOkayWords) { + if($numOkayWords && isset($booleanValues[$word])) { // make word non-required in boolean query $booleanValues[$word] = ltrim($booleanValues[$word], '+'); } else { @@ -800,6 +925,7 @@ class DatabaseQuerySelectFulltext extends Wire { return array( 'value' => trim(implode(' ', $allWords)), + 'matchValue' => trim(implode(' ', $goodWords) . ' ' . implode(' ', $altWords)), // indexable words only 'booleanValue' => trim(implode(' ', $booleanValues)), 'booleanWords' => $booleanValues, 'likeWords' => $likeWords, @@ -840,6 +966,7 @@ class DatabaseQuerySelectFulltext extends Wire { } $alternateWords = array_unique($alternateWords); + $booleanWords = $alternateWords; // prepare alternate words for inclusion in boolean value and remove any that aren’t indexable foreach($alternateWords as $key => $alternateWord) { @@ -848,21 +975,24 @@ class DatabaseQuerySelectFulltext extends Wire { if($alternateWord === $rootWord && $length > 1) { // root word is always partial match. weight less if there are other alternates to match - $less = count($alternateWords) > 1 && !empty($options['partialLess']) ? '<' : ''; - $alternateWords[$key] = $less . $alternateWord . '*'; - if($length >= $minWordLength && $length >= 3) $alternateWords[] = $less . $alternateWord; + $less = count($booleanWords) > 1 && !empty($options['partialLess']) ? '<' : ''; + $booleanWords[$key] = $less . $alternateWord . '*'; + if($length >= $minWordLength && $length >= 3) $booleanWords[] = $less . $alternateWord; + unset($alternateWords[$key]); } else if($length < $minWordLength || $this->isStopword($alternateWord)) { // alternate word not indexable, remove it unset($alternateWords[$key]); + unset($booleanWords[$key]); } else { // replace with escaped version $alternateWords[$key] = $alternateWord; + $booleanWords[$key] = $alternateWord; } } - if(!count($alternateWords)) return array(); + if(!count($booleanWords)) return array(); // rebuild boolean value to include alternates: "+(word word)" or "+word" or "" if($required) $booleanValue = ltrim($booleanValue, '+'); @@ -874,7 +1004,7 @@ class DatabaseQuerySelectFulltext extends Wire { if($booleanValue && strpos($booleanValue, '>') !== 0) $booleanValue = ">$booleanValue"; // append alternate words - $booleanValue = trim($booleanValue . ' ' . implode(' ', $alternateWords)); + $booleanValue = trim($booleanValue . ' ' . implode(' ', $booleanWords)); // package boolean value into parens and optional "+" prefix (indicating required) $booleanValue = "$required($booleanValue)"; diff --git a/wire/core/FieldSelectorInfo.php b/wire/core/FieldSelectorInfo.php index 1fff802e..0e4a673e 100644 --- a/wire/core/FieldSelectorInfo.php +++ b/wire/core/FieldSelectorInfo.php @@ -13,7 +13,7 @@ * This file is licensed under the MIT license * https://processwire.com/about/license/mit/ * - * ProcessWire 3.x, Copyright 2016 by Ryan Cramer + * ProcessWire 3.x, Copyright 2020 by Ryan Cramer * https://processwire.com * */ @@ -41,17 +41,31 @@ class FieldSelectorInfo extends Wire { * CSV keywords from schema mapped to input types to auto-determine input type from schema * */ - protected $schemaToInput = array(); - + protected $schemaToInput = array(); + /** + * Construct + * + */ public function __construct() { + $ftNops = array(); + $ftOps = Selectors::getOperators(array( + 'compareType' => Selector::compareTypeFind, + 'getValueType' => 'operator', + 'getIndexType' => 'none', + )); + + foreach($ftOps as $op) { + $ftNops[] = "!$op"; + } + $this->operators = array( 'number' => array('=', '!=', '>', '<', '>=', '<=', '=""', '!=""'), - 'text' => array('=', '!=', '%=', '^=', '$=', '=""', '!=""'), - 'fulltext' => array('%=', '*=', '~=', '^=', '$=', '=', '!=', '=""', '!=""', '!%=', '!*=', '!~=', '!^=', '!$='), + 'text' => array('=', '!=', '%=', '%^=', '%$=', '=""', '!=""'), + 'fulltext' => array_merge($ftOps, array('=', '!=', '=""', '!=""'), $ftNops), 'select' => array('=', '!=') - ); + ); $this->infoTemplate = array( // name of the field @@ -68,7 +82,7 @@ class FieldSelectorInfo extends Wire { 'options' => array(), // if field has subfields, this contains array of all above, indexed by subfield name (blank if not applicable) 'subfields' => array(), - ); + ); $this->schemaToInput = array( 'TEXT,TINYTEXT,MEDIUMTEXT,LONGTEXT,VARCHAR,CHAR' => 'text', @@ -76,8 +90,7 @@ class FieldSelectorInfo extends Wire { 'DATE' => 'date', 'INT,DECIMAL,FLOAT,DOUBLE' => 'number', 'ENUM,SET' => 'select', - ); - + ); } /** @@ -187,21 +200,19 @@ class FieldSelectorInfo extends Wire { * */ public function getOperatorLabels() { - if(empty($this->operatorLabels)) $this->operatorLabels = array( - '=' => $this->_('Equals'), - '!=' => $this->_('Not Equals'), - '>' => $this->_('Greater Than'), - '<' => $this->_('Less Than'), - '>=' => $this->_('Greater Than or Equal'), - '<=' => $this->_('Less Than or Equal'), - '%=' => $this->_('Contains Text'), - '*=' => $this->_('Contains Phrase'), - '~=' => $this->_('Contains Words'), - '^=' => $this->_('Starts With'), - '$=' => $this->_('Ends With'), - '=""' => $this->_('Is Empty'), - '!=""' => $this->_('Is Not Empty') - ); + if(!empty($this->operatorLabels)) return $this->operatorLabels; + $this->operatorLabels = Selectors::getOperators(array( + 'getIndexType' => 'operator', + 'getValueType' => 'label', + )); + $this->operatorLabels['=""'] = $this->_('Is Empty'); + $this->operatorLabels['!=""'] = $this->_('Is Not Empty'); + foreach($this->operators as $operator) { + if(isset($this->operatorLabels[$operator])) continue; + if(strpos($operator, '!') !== 0) continue; + $op = ltrim($operator, '!'); + $this->operatorLabels[$operator] = sprintf($this->_('Not: %s'), $this->operatorLabels[$op]); + } return $this->operatorLabels; } } diff --git a/wire/core/Fieldtype.php b/wire/core/Fieldtype.php index ccdb9f7a..ddba751d 100644 --- a/wire/core/Fieldtype.php +++ b/wire/core/Fieldtype.php @@ -693,7 +693,7 @@ abstract class Fieldtype extends WireData implements Module { * * #pw-group-finding * - * @param DatabaseQuerySelect $query + * @param PageFinderDatabaseQuerySelect $query * @param string $table The table name to use * @param string $subfield Name of the subfield (typically 'data', unless selector explicitly specified another) * @param string $operator The comparison operator @@ -711,6 +711,7 @@ abstract class Fieldtype extends WireData implements Module { $table = $database->escapeTable($table); $subfield = $database->escapeCol($subfield); + $operator = $database->escapeOperator($operator, WireDatabasePDO::operatorTypeComparison); $query->where("{$table}.{$subfield}{$operator}?", $value); // QA return $query; } diff --git a/wire/core/FieldtypeMulti.php b/wire/core/FieldtypeMulti.php index 7e63a6a4..beeab6fe 100644 --- a/wire/core/FieldtypeMulti.php +++ b/wire/core/FieldtypeMulti.php @@ -577,7 +577,6 @@ abstract class FieldtypeMulti extends Fieldtype { $table = $database->escapeTable($table); // note the Fulltext class can handle non-text values as well (when using non-partial text matching operators) $ft = new DatabaseQuerySelectFulltext($query); - $this->wire($ft); $ft->match($table, $col, $operator, $value); return $query; } @@ -861,7 +860,7 @@ abstract class FieldtypeMulti extends Fieldtype { * * Possible template method: If overridden, children should NOT call this parent method. * - * @param DatabaseQuerySelect $query + * @param PageFinderDatabaseQuerySelect $query * @param string $table The table name to use * @param string $subfield Name of the field (typically 'data', unless selector explicitly specified another) * @param string $operator The comparison operator @@ -878,12 +877,14 @@ abstract class FieldtypeMulti extends Fieldtype { $database = $this->wire('database'); $table = $database->escapeTable($table); - if($subfield === 'count' && (empty($value) || ctype_digit(ltrim("$value", '-'))) - && in_array($operator, array("=", "!=", ">", "<", ">=", "<="))) { + if($subfield === 'count' + && (empty($value) || ctype_digit(ltrim("$value", '-'))) + && $database->isOperator($operator, WireDatabasePDO::operatorTypeComparison)) { $value = (int) $value; $t = $table . "_" . $n; $c = $database->escapeTable($this->className()) . "_" . $n; + $operator = $database->escapeOperator($operator); $query->select("$t.num_$t AS num_$t"); $query->leftjoin( diff --git a/wire/core/PageFinder.php b/wire/core/PageFinder.php index 75c46638..72d39334 100644 --- a/wire/core/PageFinder.php +++ b/wire/core/PageFinder.php @@ -1585,6 +1585,7 @@ class PageFinder extends Wire { $q = $this->wire(new DatabaseQuerySelect()); } + /** @var PageFinderDatabaseQuerySelect $q */ $q->set('field', $field); // original field if required by the fieldtype $q->set('group', $group); // original group of the field, if required by the fieldtype $q->set('selector', $selector); // original selector if required by the fieldtype @@ -3214,3 +3215,14 @@ class PageFinder extends Wire { } } +/** + * Typehinting class for DatabaseQuerySelect object passed to Fieldtype::getMatchQuery() + * + * @property Field $field Original field + * @property string $group Original group of the field + * @property Selector $selector Original Selector object + * @property Selectors $selectors Original Selectors object + * @property DatabaseQuerySelect $parentQuery Parent database query + */ +abstract class PageFinderDatabaseQuerySelect extends DatabaseQuerySelect { } + diff --git a/wire/core/Selector.php b/wire/core/Selector.php index cf8aedfd..ef0b4f9f 100644 --- a/wire/core/Selector.php +++ b/wire/core/Selector.php @@ -1020,11 +1020,6 @@ class SelectorContainsAnyWordsExpand extends SelectorContainsAnyWords { $hasAny = true; break; } - $stem = $textTools->getWordStem($word); - if($stem && stripos($value1, $stem) !== false && preg_match('/\b' . preg_quote($stem) . '/i', $value1)) { - $hasAny = true; - break; - } $alternates = $textTools->getWordAlternates($word); foreach($alternates as $alternate) { if(stripos($value1, $alternate) && preg_match('/\b' . preg_quote($alternate) . '\b/i', $value1)) { diff --git a/wire/core/Selectors.php b/wire/core/Selectors.php index d509793f..54dd57ac 100644 --- a/wire/core/Selectors.php +++ b/wire/core/Selectors.php @@ -357,24 +357,34 @@ class Selectors extends WireArray { protected function extractOperators(&$str) { $n = 0; + $not = false; $operator = ''; $lastOperator = ''; $operators = array(); $operatorChars = self::getOperatorChars(); while(isset($str[$n]) && isset($operatorChars[$str[$n]])) { - $operator .= $str[$n]; + $c = $str[$n]; + if($operator === '!' && $c !== '=') { + // beginning of operator negation that’s not "!=" + $not = true; + $operator = ltrim($operator, '!'); + } + $operator .= $c; if(self::isOperator($operator)) { $lastOperator = $operator; } else if($lastOperator) { + if($not) $lastOperator = "!$lastOperator"; $operators[$lastOperator] = $lastOperator; $lastOperator = ''; - $operator = $str[$n]; + $operator = $c; + $not = false; } $n++; } if($lastOperator) { + if($not) $lastOperator = "!$lastOperator"; $operators[$lastOperator] = $lastOperator; } @@ -383,11 +393,14 @@ class Selectors extends WireArray { } if($operator && !isset($operators[$lastOperator])) { + // leftover characters in $operator, maybe from operator in wrong order $fail = true; if(!count($operators)) { // check if operator has a typo we can fix + // isOperator with 2nd argument true allows for and corrects some order mixups $op = self::isOperator($operator, true); if($op) { + if($not) $op = "!$op"; $operators[$op] = $op; $str = substr($str, $n); $fail = false; @@ -809,26 +822,28 @@ class Selectors extends WireArray { * Extract and return operator from end of field name, as used by selector arrays * * @param string $field - * @return bool|string + * @return array * */ - protected function getOperatorFromField(&$field) { - $operator = '='; + protected function getOperatorsFromField(&$field) { + $operators = array_keys(self::$selectorTypes); $operatorsStr = implode('', $operators); - $op = substr($field, -1); - if(strpos($operatorsStr, $op) !== false) { - // extract operator from $field + $c = substr($field, -1); + if(ctype_alnum($c)) return array('='); + + $op = ''; + while(strpos($operatorsStr, $c) !== false && strlen($field)) { + $op = $c . $op; $field = substr($field, 0, -1); - $op2 = substr($field, -1); - if(strpos($operatorsStr, $op2) !== false) { - $field = substr($field, 0, -1); - $op = $op2 . $op; - } - $operator = $op; - $field = trim($field); + $c = substr($field, -1); } - return $operator; + + if(empty($op)) return array('='); + + $operators = $this->extractOperators($op); + + return $operators; } /** @@ -863,6 +878,7 @@ class Selectors extends WireArray { foreach($data as $k => $v) { $s = $this->makeSelectorArrayItem($k, $v); $selector1 = $this->create($s['field'], $s['operator'], $s['value']); + if(!empty($s['altOperators'])) $selector1->altOperators = $s['altOperators']; $selector2 = $this->create("or$groupCnt", "=", $selector1); $selector2->quote = '('; $this->add($selector2); @@ -888,6 +904,7 @@ class Selectors extends WireArray { if($s['not']) $selector->not = true; if($s['group']) $selector->group = $s['group']; if($s['quote']) $selector->quote = $s['quote']; + if(!empty($s['altOperators'])) $selector->altOperators = $s['altOperators']; $this->add($selector); } @@ -910,7 +927,7 @@ class Selectors extends WireArray { $sanitize = 'selectorValue'; $fields = array(); $values = array(); - $operator = '='; + $operators = array('='); $whitelist = null; $not = false; $group = ''; @@ -947,7 +964,7 @@ class Selectors extends WireArray { if(isset($data['sanitizer']) && !isset($data['sanitize'])) $data['sanitize'] = $data['sanitizer']; // allow alternate if(isset($data['sanitize'])) $sanitize = $sanitizer->fieldName($data['sanitize']); - if(!empty($data['operator'])) $operator = $data['operator']; + if(!empty($data['operator'])) $operators = $this->extractOperators($data['operator']); if(!empty($data['not'])) $not = (bool) $data['not']; // may use either 'group' or 'or' to specify or-group @@ -982,7 +999,7 @@ class Selectors extends WireArray { // Non-verbose selector, where $key is the field name and $data is the value // The $key field name may have an optional operator appended to it - $operator = $this->getOperatorFromField($key); + $operators = $this->getOperatorsFromField($key); $_fields = strpos($key, '|') ? explode('|', $key) : array($key); $_values = is_array($data) ? $data : array($data); @@ -997,6 +1014,7 @@ class Selectors extends WireArray { if(count($data) == 4) { list($field, $operator, $value, $_sanitize) = $data; + $operators = $this->extractOperators($operator); if(is_array($_sanitize)) { $whitelist = $_sanitize; } else { @@ -1005,10 +1023,11 @@ class Selectors extends WireArray { } else if(count($data) == 3) { list($field, $operator, $value) = $data; + $operators = $this->extractOperators($operator); } else if(count($data) == 2) { list($field, $value) = $data; - $operator = $this->getOperatorFromField($field); + $operators = $this->getOperatorsFromField($field); } if(is_array($field)) { @@ -1024,8 +1043,10 @@ class Selectors extends WireArray { } // make sure operator is valid - if(!isset(self::$selectorTypes[$operator])) { - throw new WireException("Unrecognized selector operator '$operator'"); + foreach($operators as $operator) { + if(!isset(self::$selectorTypes[$operator])) { + throw new WireException("Unrecognized selector operator '$operator'"); + } } // determine field(s) @@ -1088,7 +1109,8 @@ class Selectors extends WireArray { return array( 'field' => count($fields) > 1 ? $fields : reset($fields), 'value' => count($values) > 1 ? $values : reset($values), - 'operator' => $operator, + 'operator' => array_shift($operators), + 'altOperators' => $operators, 'not' => $not, 'group' => $group, 'quote' => $quote, diff --git a/wire/core/WireDatabasePDO.php b/wire/core/WireDatabasePDO.php index e259f2fc..b6f449d3 100644 --- a/wire/core/WireDatabasePDO.php +++ b/wire/core/WireDatabasePDO.php @@ -21,6 +21,10 @@ */ class WireDatabasePDO extends Wire implements WireDatabase { + const operatorTypeComparison = 0; + const operatorTypeBitwise = 1; + const operatorTypeAny = 2; + /** * Log of all queries performed in this instance * @@ -89,6 +93,22 @@ class WireDatabasePDO extends Wire implements WireDatabase { */ protected $charset = ''; + /** + * Regular comparison operators + * + * @var array + * + */ + protected $comparisonOperators = array('=', '<', '>', '>=', '<=', '<>', '!='); + + /** + * Bitwise comparison operators + * + * @var array + * + */ + protected $bitwiseOperators = array('&', '~', '&~', '|', '^', '<<', '>>'); + /** * Substitute variable names according to engine as used by getVariable() method * @@ -749,46 +769,69 @@ class WireDatabasePDO extends Wire implements WireDatabase { * ~~~~~ * * @param string $str 1-2 character operator to test - * @param bool|null $bitwise NULL=allow all operators, TRUE=allow only bitwise, FALSE=do not allow bitwise (default=NULL) added 3.0.143 + * @param bool|null|int $operatorType Specify a WireDatabasePDO::operatorType* constant (3.0.162+), or any one of the following (3.0.143+): + * - `NULL`: allow all operators (default value if not specified) + * - `FALSE`: allow only comparison operators + * - `TRUE`: allow only bitwise operators + * @param bool $get Return the operator rather than true, when valid? (default=false) Added 3.0.162 * @return bool True if valid, false if not * */ - public function isOperator($str, $bitwise = null) { + public function isOperator($str, $operatorType = self::operatorTypeAny, $get = false) { - $operators = array('=', '<', '>', '>=', '<=', '<>', '!='); - $bitwiseOperators = array('&', '~', '&~', '|', '^', '<<', '>>'); $len = strlen($str); if($len > 2 || $len < 1) return false; - if($bitwise === null) { + if($operatorType === null || $operatorType === self::operatorTypeAny) { // allow all operators - $operators = array_merge($operators, $bitwiseOperators); - } else if($bitwise === true) { + $operators = array_merge($this->comparisonOperators, $this->bitwiseOperators); + + } else if($operatorType === true || $operatorType === self::operatorTypeBitwise) { // allow only bitwise operators - $operators = $bitwise; + $operators = $this->bitwiseOperators; + } else { - // false or unrecognized $bitwise value: allow only regular operators + // self::operatorTypeComparison + $operators = $this->comparisonOperators; + } + + if($get) { + $key = array_search($str, $operators, true); + return $key === false ? false : $operators[$key]; + } else { + return in_array($str, $operators, true); } - - return in_array($str, $operators, true); } /** - * Is given word a fulltext stopword to the current database engine? + * Is given word a fulltext stopword for database engine? * * @param string $word + * @param string $engine DB engine ('myisam' or 'innodb') or omit for current engine * @return bool * @since 3.0.160 * */ - public function isStopword($word) { - - if($this->engine === 'myisam') { - return DatabaseStopwords::has($word); - } - - if($this->stopwordCache === null && $this->engine === 'innodb') { + public function isStopword($word, $engine = '') { + $engine = $engine === '' ? $this->engine : strtolower($engine); + if($engine === 'myisam') return DatabaseStopwords::has($word); + if($this->stopwordCache === null) $this->getStopwords($engine, true); + return isset($this->stopwordCache[strtolower($word)]); + } + + /** + * Get all fulltext stopwords for database engine + * + * @param string $engine Specify DB engine of "myisam" or "innodb" or omit for current DB engine + * @param bool $flip Return flipped array where stopwords are array keys rather than values? for isset() use (default=false) + * @return array + * + */ + public function getStopwords($engine = '', $flip = false) { + $engine = $engine === '' ? $this->engine : strtolower($engine); + if($engine === 'myisam') return DatabaseStopwords::getAll(); + if($this->stopwordCache === null) { // && $engine === 'innodb') { $cache = $this->wire()->cache; $stopwords = null; if($cache) { @@ -804,10 +847,7 @@ class WireDatabasePDO extends Wire implements WireDatabase { } $this->stopwordCache = array_flip($stopwords); } - - if(!$this->stopwordCache) return false; - - return isset($this->stopwordCache[strtolower($word)]); + return $flip ? $this->stopwordCache : array_keys($this->stopwordCache); } /** @@ -854,6 +894,20 @@ class WireDatabasePDO extends Wire implements WireDatabase { return $this->escapeTable($table) . '.' . $this->escapeCol($col); } + /** + * Sanitize comparison operator + * + * @param string $operator + * @param bool|int|null $operatorType Specify a WireDatabasePDO::operatorType* constant (default=operatorTypeComparison) + * @param string $default Default/fallback operator to return if given one is not valid (default='=') + * @return string + * + */ + public function escapeOperator($operator, $operatorType = self::operatorTypeComparison, $default = '=') { + $operator = $this->isOperator($operator, $operatorType, true); + return $operator ? $operator : $default; + } + /** * Escape a string value, same as $db->quote() but without surrounding quotes * diff --git a/wire/modules/Fieldtype/FieldtypeText.module b/wire/modules/Fieldtype/FieldtypeText.module index 05ba679a..4ec53e00 100644 --- a/wire/modules/Fieldtype/FieldtypeText.module +++ b/wire/modules/Fieldtype/FieldtypeText.module @@ -134,7 +134,7 @@ class FieldtypeText extends Fieldtype { /** * Update a query to match the text with a fulltext index * - * @param DatabaseQuerySelect $query + * @param PageFinderDatabaseQuerySelect $query * @param string $table * @param string $subfield * @param string $operator @@ -143,8 +143,7 @@ class FieldtypeText extends Fieldtype { * */ public function getMatchQuery($query, $table, $subfield, $operator, $value) { - /** @var DatabaseQuerySelectFulltext $ft */ - $ft = $this->wire(new DatabaseQuerySelectFulltext($query)); + $ft = new DatabaseQuerySelectFulltext($query); $ft->match($table, $subfield, $operator, $value); return $query; }