Add new API methods: $pages->findRaw(), $pages->getRaw(), and $pages->getFresh()

wire/core/Pages.php

@@ -74,7 +74,6 @@
  *
  * TO-DO
  * =====
- * @todo Add a getCopy method that does a getById($id, array('cache' => false) ?
  * @todo Update saveField to accept array of field names as an option. 
  *
  */
@@ -118,7 +117,7 @@ class Pages extends Wire {
 	 * 
 	 */
 	protected $loader;
-
+	
 	/**
 	 * @var PagesEditor
 	 * 
@@ -149,6 +148,12 @@ class Pages extends Wire {
 	 */
 	protected $parents;
 
+	/**
+	 * @var PagesRaw
+	 *
+	 */
+	protected $raw;
+
 	/**
 	 * Array of PagesType managers
 	 * 
@@ -171,6 +176,7 @@ class Pages extends Wire {
 		$this->cacher = $this->wire(new PagesLoaderCache($this));
 		$this->trasher = null;
 		$this->editor = null;
+		$this->raw = null;
 	}
 	
 	/**
@@ -352,6 +358,7 @@ class Pages extends Wire {
 	 * @param array|bool|int|string $options Options to modify behavior. 
 	 *  - `verbose` (bool|int|string): Specify true to make return value array of associative arrays, each with id, parent_id, templates_id. 
 	 *    Specify integer `2` or string `*` to return verbose array of associative arrays, each with all columns from pages table. 
+	 *  - `indexed` (bool): Index by page ID? (default=false) Added 3.0.172
 	 *  - The verbose option above can also be specified as alternative to the $options argument.
 	 *  - See `Pages::find()` $options argument for additional options. 
 	 * @return array Array of page IDs, or in verbose mode: array of arrays, each with id, parent_id and templates_id keys.
@@ -376,8 +383,80 @@ class Pages extends Wire {
 		}
 		/** @var array $ids */
 		$ids = $this->find($selector, $options);
+		if(!empty($options['indexed'])) {
+			$a = array();
+			foreach($ids as $value) {
+				$id = $verbose ? $value['id'] : $value;
+				$a[$id] = $value;
+			}
+			$ids = $a;
+		}
 		return $ids;
 	}
+	
+	/**
+	 * Find pages and return raw data from them in a PHP array 
+	 * 
+	 * Note that the data returned from this method is raw and unformatted, directly
+	 * as it exists in the database. In most cases you should use `$pages->find()` instead,
+	 * but this method provides a convenient alternative for some cases. 
+	 * 
+	 * The `$selector` argument can any page-finding selector that you would provide
+	 * to a regular `$pages->find()` call. The most interesting stuff relates to the 
+	 * `$field` argument though, which is what the rest of this section looks at: 
+	 * 
+	 * If you omit the `$field` argument, it will return all data for the found pages in 
+	 * an array where the keys are the page IDs and the values are associative arrays 
+	 * containing all of the page’s raw field and property values indexed by name…
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=blog"); 
+	 * ~~~~~
+	 * …but findRaw() is more useful for cases where you want to retrieve specific things 
+	 * without having to load the entire page (or its data). Below are a few examples of
+	 * how you can do this. 
+	 * 
+	 * If you provide a string (field name) for `$field`, then it will return an array with
+	 * the values of the `data` column of that field. The `$field` can also be the name of 
+	 * a native pages table property like `id` or `name`. 
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=blog", "title"); 
+	 * ~~~~~
+	 * The above would return an array of blog page titles indexed by page ID. If you 
+	 * provide an array for `$field` then it will return an array for each page, where each
+	 * of those arrays is indexed by the field names you requested.
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=blog", [ "title", "date" ]); 
+	 * ~~~~~
+	 * You may specify field name(s) like `field.subfield` to retrieve a specific column/subfield.
+	 * When it comes to Page references or Repeaters, the subfield can also be the name of a field 
+	 * that exists on the Page reference or repeater pages. 
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=blog", [ "title", "categories.title" ]); 
+	 * ~~~~~
+	 * You can also use this format below to retrieve multiple subfields from one field:
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=blog", [ "title", "categories" => [ "id", "title" ] ]); 
+	 * ~~~~~
+	 * You may specify wildcard field name(s) like `field.*` to return all columns for `field`. 
+	 * This retrieves all columns from the field’s table. This is especially useful with fields
+	 * like Table or Combo that might have several different columns: 
+	 * ~~~~~
+	 * $a = $pages->findRaw("template=villa", "rates_table.*" ); 
+	 * ~~~~~
+	 * 
+	 * #pw-advanced
+	 * #pw-group-retrieval
+	 *
+	 * @param string|array|Selectors|int $selector Page matching selector or page ID
+	 * @param string|array|Field $field Name of field/property to get, or array of them, or omit to get all (default='')
+	 * @param array $options 
+	 * @return array
+	 * @since 3.0.172
+	 *
+	 */
+	public function findRaw($selector, $field = '', $options = array()) {
+		return $this->raw()->find($selector, $field, $options); 
+	}
 
 	/**
 	 * Returns the first page matching the given selector with no exclusions
@@ -407,6 +486,58 @@ class Pages extends Wire {
 		return $this->loader->get($selector, $options); 
 	}
 	
+	/**
+	 * Get single page and return raw data in an associative array
+	 *
+	 * Note that the data returned from this method is raw and unformatted, directly as it exists in the database. 
+	 * In most cases you should use `$pages->get()` instead, but this method is a convenient alternative for some cases. 
+	 *
+	 * Please see the documentation for the `$pages->findRaw()` method, which all applies to this method as well.
+	 * The biggest difference is that this method returns data for just 1 page, unlike `$pages->findRaw()` which can
+	 * return data for many pages at once. 
+	 * 
+	 * #pw-advanced
+	 *
+	 * @param string|array|Selectors|int $selector Page matching selector or page ID
+	 * @param string|array|Field $field Name of field/property to get, or array of them, or omit to get all (default='')
+	 * @param array $options
+	 * @return array
+	 *
+	 */
+	public function getRaw($selector, $field = '', $options = array()) {
+		return $this->raw()->get($selector, $field, $options);
+	}
+	
+	/**
+	 * Get a fresh, non-cached copy of a Page from the database
+	 *
+	 * This method is the same as `$pages->get()` except that it skips over all memory caches when loading a Page.
+	 * Meaning, if the Page is already in memory, it doesn’t use the one in memory and instead reloads from the DB.
+	 * Nor does it place the Page it loads in any memory cache. Use this method to load a fresh copy of a page
+	 * that you might need to compare to an existing loaded copy, or to load a copy that won’t be seen or touched
+	 * by anything in ProcessWire other than your own code.
+	 *
+	 * ~~~~~
+	 * $p1 = $pages->get(1234);
+	 * $p2 = $pages->get($p1->path);
+	 * $p1 === $p2; // true: same Page instance
+	 *
+	 * $p3 = $pages->getFresh($p1);
+	 * $p1 === $p3; // false: same Page but different instance
+	 * ~~~~~
+	 *
+	 * #pw-advanced
+	 *
+	 * @param Page|string|array|Selectors|int $selectorOrPage Specify Page to get copy of, selector or ID
+	 * @param array $options Options to modify behavior
+	 * @return Page|NullPage
+	 * @since 3.0.172
+	 *
+	 */
+	public function getFresh($selectorOrPage, $options = array()) {
+		return $this->loader()->getFresh($selectorOrPage, $options);
+	}
+
 	/**
 	 * Get one ID of page matching given selector with no exclusions, like get() but returns ID rather than a Page
 	 *
@@ -1562,6 +1693,18 @@ class Pages extends Wire {
 		return $this->parents;
 	}
 
+	/**
+	 * @return PagesRaw
+	 * @since 3.0.172
+	 *
+	 * #pw-internal
+	 *
+	 */
+	public function raw() {
+		if(!$this->raw) $this->raw = $this->wire(new PagesRaw($this));
+		return $this->raw;
+	}
+
 	/**
 	 * Get array of all PagesType managers
 	 * 

wire/core/PagesRaw.php

@@ -0,0 +1,734 @@
+<?php namespace ProcessWire;
+
+/**
+ * ProcessWire Pages Raw Tools
+ *
+ * ProcessWire 3.x, Copyright 2021 by Ryan Cramer
+ * https://processwire.com
+ *
+ */
+
+class PagesRaw extends Wire {
+
+	/**
+	 * @var Pages
+	 *
+	 */
+	protected $pages;
+
+	/**
+	 * Construct
+	 *
+	 * @param Pages $pages
+	 *
+	 */
+	public function __construct(Pages $pages) {
+		$this->pages = $pages;
+	}
+
+	/**
+	 * Find pages and return raw data from them in a PHP array
+	 *
+	 * @param string|array|Selectors $selector
+	 * @param string|Field|int|array $field Field/property name to get or array of them (or omit to get all)
+	 * @param array $options See options for Pages::find
+	 *  - `indexed` (bool): Index by page ID? (default=false)
+	 * @return array
+	 * @since 3.0.172
+	 *
+	 */
+	public function find($selector, $field = '', $options = array()) {
+		if(!is_array($options)) $options = array('indexed' => (bool) $options);
+		$finder = new PagesRawFinder($this->pages);
+		$this->wire($finder);
+		return $finder->find($selector, $field, $options);
+	}
+
+	/**
+	 * Get page (no exclusions) and return raw data from it in a PHP array
+	 *
+	 * @param string|array|Selectors $selector
+	 * @param string|Field|int|array $field Field/property name to get or array of them (or omit to get all)
+	 * @param array $options See options for Pages::find
+	 *  - `indexed` (bool): Index by page ID? (default=false)
+	 * @return array
+	 * @since 3.0.172
+	 *
+	 */
+	public function get($selector, $field = '', $options = array()) {
+		if(!is_array($options)) $options = array('indexed' => (bool) $options);
+		$options['findOne'] = true;
+		if(!isset($options['findAll'])) $options['findAll'] = true;
+		$values = $this->find($selector, $field, $options);
+		return reset($values);
+	}
+	
+}
+
+/**
+ * ProcessWire Pages Raw Finder
+ *
+ */
+class PagesRawFinder extends Wire {
+	
+	/**
+	 * @var Pages
+	 *
+	 */
+	protected $pages;
+
+	/**
+	 * @var array
+	 * 
+	 */
+	protected $options = array();
+
+	/**
+	 * @var array
+	 * 
+	 */
+	protected $defaults = array(
+		'indexed' => true,
+		'findOne' => false,
+	);
+
+	/**
+	 * @var string|array|Selectors
+	 * 
+	 */
+	protected $selector = '';
+
+	/**
+	 * @var array
+	 * 
+	 */
+	protected $requestFields = array();
+
+	/**
+	 * @var array
+	 * 
+	 */
+	protected $nativeFields = array();
+
+	/**
+	 * @var array
+	 *
+	 */
+	protected $customFields = array();
+
+	/**
+	 * @var array
+	 *
+	 */
+	protected $customCols = array();
+
+	/**
+	 * Columns requested as fieldName.col rather than fieldName[col]
+	 * 
+	 * (not currently accounted for, future use)
+	 * 
+	 * @var array
+	 * 
+	 */
+	protected $customDotCols = array();
+
+	/**
+	 * Results of the raw find
+	 * 
+	 * @var array
+	 * 
+	 */
+	protected $values = array();
+
+	/**
+	 * True to return array indexed by field name for each page, false to return single value for each page
+	 * 
+	 * @var bool
+	 * 
+	 */
+	protected $getMultiple = true;
+
+	/**
+	 * Get all data for pages?
+	 * 
+	 * @var bool
+	 * 
+	 */
+	protected $getAll = false;
+
+	/**
+	 * IDs of pages to find, becomes array once known
+	 * 
+	 * @var null|array
+	 * 
+	 */
+	protected $ids = null;
+	
+	/**
+	 * Construct
+	 *
+	 * @param Pages $pages
+	 *
+	 */
+	public function __construct(Pages $pages) {
+		$this->pages = $pages;
+	}
+
+	/**
+	 * @param string|int|array|Selectors
+	 * @param string|array|Field $field
+	 * @param array $options
+	 * 
+	 */
+	protected function init($selector, $field, $options) {
+
+		$this->selector = $selector;
+		$this->options = array_merge($this->defaults, $options);
+		$this->values = array();
+		$this->requestFields = array();
+		$this->customFields = array();
+		$this->nativeFields = array();
+		$this->customCols = array();
+		$this->getMultiple = true;
+		$this->getAll = false;
+		$this->ids = null;
+		
+		if(empty($field)) {
+			$this->getAll = true;
+			
+		} else if(is_string($field) && strpos($field, ',') !== false) {
+			// multiple fields requested in CSV string, we will return an array for each page
+			$this->requestFields = explode(',', $field);
+			
+		} else if(is_array($field)) {
+			// one or more fields requested in array, we wil return an array for each page
+			$this->requestFields = $field;
+			
+		} else {
+			// one field requested in string or Field object
+			$this->requestFields = array($field);
+			$this->getMultiple = false;
+		}
+
+		if(!$this->getAll) {
+			// split request fields into nativeFields and customFields
+			$this->splitFields();
+		}
+	}
+
+	/**
+	 * Find pages and return raw data from them in a PHP array
+	 *
+	 * How to use the `$field` argument:
+	 *
+	 * - If you provide an array for $field then it will return an array for each page, indexed by
+	 *   the field names you requested.
+	 *
+	 * - If you provide a string (field name) or Field object, then it will return an array with
+	 *   the values of the 'data' column of that field.
+	 *
+	 * - You may request field name(s) like `field.subfield` to retrieve a specific column/subfield.
+	 *
+	 * - You may request field name(s) like `field.*` to return all columns/subfields for `field`,
+	 *   in this case, an associative array value will be returned for each page.
+	 *
+	 * @param string|array|Selectors $selector
+	 * @param string|Field|int|array $field Field/property name or array of of them
+	 * @param array $options See options for Pages::find
+	 * @return array
+	 * @since 3.0.172
+	 *
+	 */
+	public function find($selector, $field = '', $options = array()) {
+
+		$this->init($selector, $field, $options);
+		
+		// requested native pages table fields/properties
+		if(count($this->nativeFields) || $this->getAll) {
+			// one or more native pages table column(s) requested
+			$this->findNativeFields();
+		}
+		
+		// requested custom fields
+		if(count($this->customFields) || $this->getAll) {
+			$this->findCustom();
+		}
+
+		// reduce return value when expected
+		if(!$this->getMultiple) {
+			foreach($this->values as $id => $row) {
+				$this->values[$id] = reset($row);
+			}
+		}
+		
+		if(!$this->options['indexed']) {
+			$this->values = array_values($this->values);
+		}
+
+		return $this->values;
+	}
+	
+	/**
+	 * Split requestFields into native and custom field arrays
+	 * 
+	 * Populates $this->nativeFields, $this->customFields, $this->customCols
+	 * 
+	 */
+	protected function splitFields() {
+		
+		$fields = $this->wire()->fields;
+
+		// split request fields into custom fields and native (pages table) fields
+		foreach($this->requestFields as $key => $fieldName) {
+			
+			if(empty($fieldName)) continue;
+			if(is_string($fieldName)) $fieldName = trim($fieldName);
+			$colName = '';
+			$dotCol = false;
+			
+			if($fieldName === '*') {
+				// get all (not yet supported)
+				$fieldObject = null;
+				
+			} else if($fieldName instanceof Field) {
+				// Field object
+				$fieldObject = $fieldName;
+				
+			} else if(is_array($fieldName)) {
+				// Array where [ 'field' => [ 'subfield'' ]] 
+				$colName = $fieldName; // array
+				$fieldName = $key;
+				$fieldObject = isset($this->customFields[$fieldName]) ? $this->customFields[$fieldName] : null;
+				if(!$fieldObject) $fieldObject = $fields->get($fieldName);
+				if(!$fieldObject) continue;
+				
+			} else if(is_int($fieldName) || ctype_digit("$fieldName")) {
+				// Field ID
+				$fieldObject = $fields->get((int) $fieldName);
+				
+			} else if(is_string($fieldName)) {
+				// Field name, subfield/column may optionally be specified as field.subfield
+				if(strpos($fieldName, '.')) {
+					list($fieldName, $colName) = explode('.', $fieldName, 2);
+					$dotCol = true;
+				} else if(strpos($fieldName, '[')) {
+					list($fieldName, $colName) = explode('[', $fieldName, 2); 
+					$colName = rtrim($colName, ']');
+				}
+				$fieldObject = isset($this->customFields[$fieldName]) ? $this->customFields[$fieldName] : null;
+				if(!$fieldObject) $fieldObject = $fields->get($fieldName);
+				
+			} else {
+				// something we do not recognize
+				continue;
+			}
+			
+			if($fieldObject instanceof Field) {
+				$this->customFields[$fieldName] = $fieldObject;
+				if(!empty($colName)) {
+					$colNames = is_array($colName) ? $colName : array($colName);
+					foreach($colNames as $col) {
+						if(!isset($this->customCols[$fieldName])) $this->customCols[$fieldName] = array();
+						$this->customCols[$fieldName][$col] = $col;
+						if($dotCol) {
+							if(!isset($this->customDotCols[$fieldName])) $this->customDotCols[$fieldName] = array();
+							$this->customDotCols[$fieldName][$col] = $col;
+						}
+					}
+				}
+			} else {
+				$this->nativeFields[$fieldName] = $fieldName;
+			}
+		}
+	}
+
+	/**
+	 * Find raw native fields
+	 *
+	 */
+	protected function findNativeFields() {
+		
+		$this->ids = array();
+		$allNatives = array();
+		
+		foreach($this->findIDs($this->selector, '*') as $row) {
+			$id = (int) $row['id'];
+			$this->ids[$id] = $id;
+			$this->values[$id] = isset($this->values[$id]) ? array_merge($this->values[$id], $row) : $row;
+			if(empty($allNatives)) {
+				foreach(array_keys($row) as $key) {
+					$allNatives[$key] = $key;
+				}
+			}
+		}
+		
+		if(!count($this->values)) return;
+		
+		if($this->getAll) $this->nativeFields = $allNatives;
+
+		// native columns we will populate into $values
+		$getNatives = array();
+
+		foreach($this->nativeFields as $fieldName) {
+			if($fieldName === '*' || $fieldName === 'pages' || $fieldName === 'pages.*') {
+				// get all columns
+				$colName = '';
+			} else if(strpos($fieldName, 'pages.') === 0) {
+				// pages table column requested by name
+				list(,$colName) = explode('.', $fieldName, 2);
+			} else {
+				// column requested by name on its own
+				$colName = $fieldName;
+			}
+			if(empty($colName)) {
+				// get all native pages table columns
+				$getNatives = $allNatives;
+			} else if(isset($allNatives[$colName])) {
+				// get specific native pages table columns
+				$getNatives[$colName] = $colName;
+			} else {
+				// fieldName is not a known field or pages column
+			}
+		}
+
+		if(count($getNatives)) {
+			// remove any native data that is present but was not requested
+			foreach($this->values as $id => $row) {
+				foreach($row as $colName => $value) {
+					if(!isset($getNatives[$colName])) {
+						unset($this->values[$id][$colName]);
+					}
+				}
+			}
+		}
+	}
+
+	/**
+	 * Gateway to finding custom fields whether specific, all or none
+	 * 
+	 */
+	protected function findCustom() {
+		if(count($this->customFields)) {
+			// one or more custom fields requested
+			if($this->ids === null) {
+				// only find IDs if we didn’t already in the nativeFields section
+				$this->ids = $this->findIDs($this->selector, false);
+			}
+			if(!count($this->ids)) return;
+			foreach($this->customFields as $fieldName => $field) {
+				/** @var Field $field */
+				$cols = isset($this->customCols[$fieldName]) ? $this->customCols[$fieldName] : array();
+				$this->findCustomField($field, $cols);
+			}
+		} else if($this->getAll && !empty($this->ids)) {
+			$this->findCustomAll();
+		}
+	}
+
+	/**
+	 * Find raw custom field
+	 *
+	 * @param Field $field
+	 * @param array $cols
+	 * @throws WireException
+	 *
+	 */
+	protected function findCustomField(Field $field, array $cols) {
+
+		$database = $this->wire()->database;
+		$sanitizer = $this->wire()->sanitizer;
+		$getArray = true;
+		$getCols = array();
+		$skipCols = array();
+		$getAllCols = false;
+		$pageRefCols = array();
+
+		/** @var FieldtypeMulti $fieldtypeMulti */
+		$fieldtype = $field->type;
+		$fieldtypeMulti = $field->type instanceof FieldtypeMulti ? $fieldtype : null;
+		$fieldtypePage = $fieldtype instanceof FieldtypePage ? $fieldtype : null;
+		$fieldtypeRepeater = $fieldtype instanceof FieldtypeRepeater ? $fieldtype : null;
+		
+		$fieldName = $field->name;
+		$schema = $fieldtype->getDatabaseSchema($field);
+		$schema = $fieldtype->trimDatabaseSchema($schema, array('trimDefault' => false));
+		$table = $database->escapeTable($field->getTable());
+		$sorts = array();
+
+		if(empty($table)) return;
+
+		if(empty($cols)) { 
+			// no cols specified
+			$trimSchema = $fieldtype->trimDatabaseSchema($schema, array('trimDefault' => true, 'trimMeta' => true));
+			unset($trimSchema['data']); 
+			foreach($trimSchema as $key => $value) {
+				// multi-language columns do not count as custom schema
+				if(strpos($key, 'data') === 0 && ctype_digit(substr($key, 4))) unset($trimSchema[$key]); 
+			}	
+			if(empty($trimSchema)) {
+				// if table doesn’t maintain a custom schema, just get data column
+				$getArray = false;
+				$getCols[] = 'data';
+			} else {
+				// table maintains custom schema, get all columns
+				$getAllCols = true;
+				$skipCols[] = 'pages_id';
+			}
+			
+		} else if(reset($cols) === '*') {
+			$getAllCols = true;
+			
+		} else {
+			foreach($cols as $key => $col) {
+				$col = $sanitizer->fieldName($col);
+				$col = $database->escapeCol($col);
+				if(isset($schema[$col])) {
+					$getCols[$col] = $col;
+				} else if($fieldtypePage || $fieldtypeRepeater) {
+					$pageRefCols[$col] = $col;
+				} else {
+					// unknown column
+				}
+			}
+			if(count($pageRefCols)) {
+				// get just the data column when a field within a Page reference is asked for
+				$getCols['data'] = 'data';
+			}
+			if(count($getCols) === 1 && !$this->getMultiple) {
+				// if only getting single field we will populate its value rather than 
+				// its value in an associative array
+				$getArray = false;
+			}
+		}
+
+		if($fieldtypeMulti) {
+			$orderByCols = $fieldtypeMulti->get('orderByCols');
+			if($fieldtypeMulti->useOrderByCols && !empty($orderByCols)) {
+				foreach($orderByCols as $key => $col) {
+					$desc = strpos($col, '-') === 0 ? ' DESC' : '';
+					$col = $sanitizer->fieldName(ltrim($col, '-'));
+					if(!array_key_exists($col, $schema)) continue;
+					$sorts[$key] = '`' . $database->escapeCol($col) . '`' . $desc;
+				}
+			}
+			if(empty($sorts) && isset($schema['sort'])) {
+				$sorts[] = "`sort`";
+			}
+		}
+
+		$this->ids(true); // converts this->ids to CSV string
+		$idsCSV = &$this->ids;
+		$colSQL = $getAllCols ? '*' : '`' . implode('`,`', $getCols) . '`';
+		if(!$getAllCols && !in_array('pages_id', $getCols)) $colSQL .= ',`pages_id`';
+		$sql = "SELECT $colSQL FROM `$table` WHERE pages_id IN($idsCSV) ";
+		if(count($sorts)) $sql .= "ORDER BY " . implode(',', $sorts);
+
+		$query = $database->prepare($sql);
+		$query->execute();
+
+		while($row = $query->fetch(\PDO::FETCH_ASSOC)) {
+			
+			$id = $row['pages_id'];
+			
+			if(!$getAllCols && !isset($getCols['pages_id'])) unset($row['pages_id']);
+			
+			foreach($skipCols as $skipCol) {
+				unset($row[$skipCol]);
+			}
+			
+			if($getAllCols) {
+				$value = $row;
+			} else if($getArray) {
+				$value = array();
+				foreach($getCols as $col) {
+					$value[$col] = isset($row[$col]) ? $row[$col] : null;
+				}
+			} else {
+				$col = reset($getCols);
+				if(empty($col)) $col = 'data';
+				$value = $row[$col];
+			}
+			
+			if(!isset($this->values[$id])) {
+				// Overall page placeholder array
+				$this->values[$id] = array();
+			}
+		
+			if($fieldtypeMulti) {
+				// FieldtypeMulti types may contain multiple rows
+
+				/** @var FieldtypeMulti $fieldtype */
+				if(!isset($this->values[$id][$fieldName])) {
+					$this->values[$id][$fieldName] = array();
+				}
+
+				if($fieldtypePage && count($pageRefCols)) {
+					// reduce page reference to just the IDs, indexed by IDs
+					if(isset($value['data'])) $value = $value['data'];
+					$this->values[$id][$fieldName][$value] = $value;
+				} else {
+					$this->values[$id][$fieldName][] = $value;
+				}
+				
+			} else if($fieldtypeRepeater && count($pageRefCols)) {
+				$repeaterIds = isset($value['data']) ? explode(',', $value['data']) : explode(',', $value);
+				foreach($repeaterIds as $repeaterId) {
+					$this->values[$id][$fieldName][$repeaterId] = $repeaterId;
+				}
+				
+			} else {
+				$this->values[$id][$fieldName] = $value;
+			}
+		}
+
+		$query->closeCursor();
+
+		if(count($pageRefCols)) {
+			if($fieldtypePage) {
+				$this->findCustomFieldtypePage($field, $fieldName, $pageRefCols);
+			} else if($fieldtypeRepeater) {
+				$this->findCustomFieldtypePage($field, $fieldName, $pageRefCols);
+			}
+		}
+	}
+	
+	protected function findCustomFieldtypePage(Field $field, $fieldName, array $pageRefCols) {
+		// print_r($values);
+		$pageRefIds = array();
+		foreach($this->values as $pageId => $row) {
+			if(!isset($row[$fieldName])) continue;
+			$pageRefIds = array_merge($pageRefIds, $row[$fieldName]);
+		}
+
+		if(!$this->getMultiple && count($pageRefCols) === 1) {
+			$pageRefCols = implode('', $pageRefCols);
+		}
+
+		$pageRefIds = array_unique($pageRefIds);
+		$finder = new PagesRawFinder($this->pages);
+		$this->wire($finder);
+		$options = $this->options;
+		$options['indexed'] = true;
+		$pageRefRows = $finder->find($pageRefIds, $pageRefCols, $options);
+
+		foreach($this->values as $pageId => $pageRow) {
+			if(!isset($pageRow[$fieldName])) continue;
+			foreach($pageRow[$fieldName] as $pageRefId) {
+				$this->values[$pageId][$fieldName][$pageRefId] = $pageRefRows[$pageRefId];
+			}
+			if(!$this->getMultiple && $field->get('derefAsPage') > 0) {
+				$this->values[$pageId][$fieldName] = reset($this->values[$pageId][$fieldName]); 
+			} else if(empty($this->options['indexed'])) {
+				$this->values[$pageId][$fieldName] = array_values($this->values[$pageId][$fieldName]);
+			}
+		}
+	}
+
+	/**
+	 * Find/populate all custom fields
+	 * 
+	 */
+	protected function findCustomAll() {
+		
+		$idsByTemplate = array();
+		
+		foreach($this->ids() as $id) {
+			if(!isset($this->values[$id])) continue;
+			$row = $this->values[$id];
+			$templateId = $row['templates_id'];
+			if(!isset($idsByTemplate[$templateId])) $idsByTemplate[$templateId] = array();
+			$idsByTemplate[$templateId][$id] = $id;
+		}
+		
+		foreach($idsByTemplate as $templateId => $pageIds) {
+			$template = $this->wire()->templates->get($templateId);
+			if(!$template) continue;
+			foreach($template->fieldgroup as $field) {
+				$this->findCustomField($field, array()); 
+			}
+		}
+	}
+
+	/**
+	 * Front-end to pages.findIDs that optionally accepts array of page IDs
+	 * 
+	 * @param array|string|Selectors $selector
+	 * @param bool|string $verbose One of true, false, or '*'
+	 * @param array $options
+	 * @return array
+	 * @throws WireException
+	 * 
+	 */
+	protected function findIDs($selector, $verbose, array $options = array()) {
+		
+		$options = array_merge($this->options, $options); 
+		$options['verbose'] = $verbose;
+		$options['indexed'] = true;
+	
+		// if selector was just a page ID, return it in an id indexed array
+		if(is_int($selector) || (is_string($selector) && ctype_digit($selector))) {
+			$id = (int) $selector;
+			return array($id => $id); 
+		}
+
+		// if selector is not array of page IDs then let pages.findIDs handle it
+		if(!is_array($selector) || !isset($selector[0]) || !ctype_digit((string) $selector[0])) {
+			return $this->pages->findIDs($selector, $options);
+		}
+		
+		// at this point selector is an array of page IDs
+
+		if(empty($verbose)) {
+			// if selector already has what is needed and verbose data not needed,
+			// then return it now, but make sure it is indexed by ID first
+			$a = array();
+			foreach($selector as $id) $a[(int) $id] = (int) $id;
+			return $a;
+		}
+
+		// convert selector to CSV string of page IDs
+		$selector = implode(',', array_map('intval', $selector));
+	
+		if($verbose === '*') {
+			// get all columns
+			$sql = "SELECT * FROM pages WHERE id IN($selector)";
+		} else {
+			// get just base columns
+			$sql = "SELECT id, templates_id, parent_id FROM pages WHERE id IN($selector)";
+		}
+		
+		$query = $this->wire()->database->prepare($sql);
+		$query->execute();
+		$rows = array();
+		
+		while($row = $query->fetch(\PDO::FETCH_ASSOC)) {
+			$id = (int) $row['id'];
+			$rows[$id] = $row;
+		}
+		
+		$query->closeCursor();
+		
+		return $rows;
+	}
+
+	/**
+	 * Get or convert $this->ids to/from CSV
+	 * 
+	 * The point of this is just to minimize the quantity of copies of IDs we are keeping around. 
+	 * In case the quantity gets to be huge, it'll be more memory friendly. 
+	 * 
+	 * @param bool $csv
+	 * @return array|string
+	 * 
+	 */
+	protected function ids($csv = false) {
+		if($this->ids === null) return $csv ? '' : array();
+		if($csv) {
+			if(is_array($this->ids)) $this->ids = implode(',', array_map('intval', $this->ids));
+		} else if(is_string($this->ids)) {
+			// this likely cannot occur with current logic but here in case that changes
+			$this->ids = explode(',', $this->ids);
+		}
+		return $this->ids;
+	}
+}

wire/modules/Process/ProcessModule/ProcessModule.module

@@ -351,7 +351,7 @@ class ProcessModule extends Process {
 		}
 
 		if($input->get('reset') == 1) {
-			$modules->resetCache();
+			$modules->refresh(true);
 			$this->message(sprintf($this->_('Modules cache refreshed (%d modules)'), count($modules)));
 			$edit = $input->get->fieldName('edit');
 			$duplicates = $modules->duplicates()->getDuplicates();
@@ -1227,7 +1227,7 @@ class ProcessModule extends Process {
 		}
 		
 		$this->session->CSRF->validate();
-		$this->modules->resetCache();
+		$this->modules->refresh();
 
 		$url = $this->session->get('ProcessModuleDownloadURL');
 		$className = $this->session->get('ProcessModuleClassName');
@@ -1395,7 +1395,7 @@ class ProcessModule extends Process {
 		}
 		
 		if($input->get('refresh') == $moduleName) {
-			$modules->resetCache();
+			$modules->refresh();
 			$session->redirect("./edit?name=$moduleName$collapseInfo");
 			return '';
 		}