[ticket/14257] Add text_reparser manager

PHPBB3-14257

phpBB/config/default/container/services_console.yml

@@ -187,9 +187,9 @@ services:
         class: phpbb\console\command\reparser\reparse
         arguments:
             - @user
-            - @text_reparser_collection
-            - @config_text
             - @text_reparser.lock
+            - @text_reparser_collection
+            - @text_reparser.manager
         tags:
             - { name: console.command }
 

phpBB/config/default/container/services_cron.yml

@@ -153,6 +153,7 @@ services:
             - @config
             - @config_text
             - @text_reparser.lock
+            - @text_reparser.manager
             - @text_reparser_collection
         calls:
             - [set_name, [cron.task.text_reparser.pm_text]]
@@ -166,6 +167,7 @@ services:
             - @config
             - @config_text
             - @text_reparser.lock
+            - @text_reparser.manager
             - @text_reparser_collection
         calls:
             - [set_name, [cron.task.text_reparser.poll_option]]
@@ -179,6 +181,7 @@ services:
             - @config
             - @config_text
             - @text_reparser.lock
+            - @text_reparser.manager
             - @text_reparser_collection
         calls:
             - [set_name, [cron.task.text_reparser.poll_title]]
@@ -192,6 +195,7 @@ services:
             - @config
             - @config_text
             - @text_reparser.lock
+            - @text_reparser.manager
             - @text_reparser_collection
         calls:
             - [set_name, [cron.task.text_reparser.post_text]]
@@ -205,6 +209,7 @@ services:
             - @config
             - @config_text
             - @text_reparser.lock
+            - @text_reparser.manager
             - @text_reparser_collection
         calls:
             - [set_name, [cron.task.text_reparser.user_signature]]

phpBB/config/default/container/services_text_reparser.yml

@@ -1,4 +1,11 @@
 services:
+    text_reparser.manager:
+        class: phpbb\textreparser\manager
+        arguments:
+            - @config
+            - @config_text
+            - @text_reparser_collection
+
     text_reparser.lock:
         class: phpbb\lock\db
         arguments:

phpBB/phpbb/console/command/reparser/reparse.php

@@ -22,11 +22,6 @@ use Symfony\Component\Console\Style\SymfonyStyle;
 
 class reparse extends \phpbb\console\command\command
 {
-	/**
-	* @var \phpbb\config\db_text
-	*/
-	protected $config_text;
-
 	/**
 	* @var InputInterface
 	*/
@@ -47,6 +42,11 @@ class reparse extends \phpbb\console\command\command
 	 */
 	protected $reparse_lock;
 
+	/**
+	 * @var \phpbb\textreparser\manager
+	 */
+	protected $reparser_manager;
+
 	/**
 	* @var \phpbb\di\service_collection
 	*/
@@ -65,13 +65,13 @@ class reparse extends \phpbb\console\command\command
 	* @param \phpbb\config\db_text $config_text
 	* @param \phpbb\lock\db $reparse_lock
 	*/
-	public function __construct(\phpbb\user $user, \phpbb\di\service_collection $reparsers, \phpbb\config\db_text $config_text, \phpbb\lock\db $reparse_lock)
+	public function __construct(\phpbb\user $user, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers)
 	{
 		require_once __DIR__ . '/../../../../includes/functions_content.php';
 
-		$this->config_text = $config_text;
-		$this->reparsers = $reparsers;
 		$this->reparse_lock = $reparse_lock;
+		$this->reparser_manager = $reparser_manager;
+		$this->reparsers = $reparsers;
 		parent::__construct($user);
 	}
 
@@ -177,8 +177,6 @@ class reparse extends \phpbb\console\command\command
 			throw new runtime_exception('REPARSE_LOCK_ERROR', array(), null, 1);
 		}
 
-		$this->load_resume_data();
-
 		$name = $input->getArgument('reparser-name');
 		if (isset($name))
 		{
@@ -233,15 +231,6 @@ class reparse extends \phpbb\console\command\command
 		return $value;
 	}
 
-	/**
-	* Load the resume data from the database
-	*/
-	protected function load_resume_data()
-	{
-		$resume_data = $this->config_text->get('reparser_resume');
-		$this->resume_data = (empty($resume_data)) ? array() : unserialize($resume_data);
-	}
-
 	/**
 	* Reparse all text handled by given reparser within given range
 	*
@@ -250,6 +239,7 @@ class reparse extends \phpbb\console\command\command
 	protected function reparse($name)
 	{
 		$reparser = $this->reparsers[$name];
+		$this->resume_data = $this->reparser_manager->get_resume_data($name);
 		if ($this->input->getOption('dry-run'))
 		{
 			$reparser->disable_save();
@@ -288,34 +278,10 @@ class reparse extends \phpbb\console\command\command
 			$current = $start - 1;
 			$progress->setProgress($max + 1 - $start);
 
-			$this->update_resume_data($name, $current);
+			$this->reparser_manager->update_resume_data($name, $min, $current, $size, !$this->input->getOption('dry-run'));
 		}
 		$progress->finish();
 
 		$this->io->newLine(2);
 	}
-
-	/**
-	* Save the resume data to the database
-	*/
-	protected function save_resume_data()
-	{
-		$this->config_text->set('reparser_resume', serialize($this->resume_data));
-	}
-
-	/**
-	* Save the resume data to the database
-	*
-	* @param string $name    Reparser name
-	* @param string $current Current ID
-	*/
-	protected function update_resume_data($name, $current)
-	{
-		$this->resume_data[$name] = array(
-			'range-min'  => $this->get_option($name, 'range-min'),
-			'range-max'  => $current,
-			'range-size' => $this->get_option($name, 'range-size'),
-		);
-		$this->save_resume_data();
-	}
 }

phpBB/phpbb/cron/task/text_reparser/reparser.php

@@ -37,15 +37,20 @@ class reparser extends \phpbb\cron\task\base
 	protected $reparse_lock;
 
 	/**
-	 * @var \phpbb\di\service_collection
+	 * @var \phpbb\textreparser\manager
 	 */
-	protected $reparsers;
+	protected $reparser_manager;
 
 	/**
 	 * @var string
 	 */
 	protected $reparser_name;
 
+	/**
+	 * @var \phpbb\di\service_collection
+	 */
+	protected $reparsers;
+
 	/**
 	 * @var array
 	 */
@@ -57,13 +62,15 @@ class reparser extends \phpbb\cron\task\base
 	 * @param \phpbb\config\config			$config
 	 * @param \phpbb\config\db_text			$config_text
 	 * @param \phpbb\lock\db				$reparse_lock
+	 * @param \phpbb\textreparser\manager	$reparser_manager
 	 * @param \phpbb\di\service_collection	$reparsers
 	 */
-	public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\lock\db $reparse_lock, \phpbb\di\service_collection $reparsers)
+	public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers)
 	{
 		$this->config = $config;
 		$this->config_text = $config_text;
 		$this->reparse_lock = $reparse_lock;
+		$this->reparser_manager = $reparser_manager;
 		$this->reparsers = $reparsers;
 	}
 
@@ -74,18 +81,11 @@ class reparser extends \phpbb\cron\task\base
 	 */
 	public function set_reparser($reparser)
 	{
-		if (isset($this->reparsers[$reparser]))
+		$this->reparser_name = (!isset($this->reparsers[$reparser]) ? 'text_reparser.' : '') . $reparser;
-		{
-			$this->reparser_name = preg_replace('(^text_reparser\\.)', '', $reparser);
-		}
-		else if (isset($this->reparsers['text_reparser.' . $reparser]))
-		{
-			$this->reparser_name = $reparser;
-		}
 
 		if ($this->resume_data === null)
 		{
-			$this->load_resume_data();
+			$this->reparser_manager->get_resume_data($this->reparser_name);
 		}
 	}
 
@@ -96,10 +96,10 @@ class reparser extends \phpbb\cron\task\base
 	{
 		if ($this->resume_data === null)
 		{
-			$this->load_resume_data();
+			$this->reparser_manager->get_resume_data($this->reparser_name);
 		}
 
-		if (empty($this->resume_data[$this->reparser_name]['range-max']) || $this->resume_data[$this->reparser_name]['range-max'] >= $this->resume_data[$this->reparser_name]['range-min'])
+		if (empty($this->resume_data['range-max']) || $this->resume_data['range-max'] >= $this->resume_data['range-min'])
 		{
 			return true;
 		}
@@ -122,7 +122,12 @@ class reparser extends \phpbb\cron\task\base
 			}
 		}
 
-		return true;
+		if ($this->config[$this->reparser_name . '_cron_interval'] != -1)
+		{
+			return $this->config[$this->reparser_name . '_last_cron'] < time() - $this->config[$this->reparser_name . '_cron_interval'];
+		}
+
+		return false;
 	}
 
 	/**
@@ -134,17 +139,17 @@ class reparser extends \phpbb\cron\task\base
 		{
 			if ($this->resume_data === null)
 			{
-				$this->load_resume_data();
+				$this->resume_data = $this->reparser_manager->get_resume_data($this->reparser_name);
 			}
 
 			/**
 			 * @var \phpbb\textreparser\reparser_interface $reparser
 			 */
-			$reparser = isset($this->reparsers[$this->reparser_name]) ? $this->reparsers[$this->reparser_name] : $this->reparsers['text_reparser.' . $this->reparser_name];
+			$reparser = $this->reparsers[$this->reparser_name];
 
-			$min = !empty($this->resume_data[$this->reparser_name]['range-min']) ? $this->resume_data[$this->reparser_name]['range-min'] : self::MIN;
+			$min = !empty($this->resume_data['range-min']) ? $this->resume_data['range-min'] : self::MIN;
-			$current = !empty($this->resume_data[$this->reparser_name]['range-max']) ? $this->resume_data[$this->reparser_name]['range-max'] : $reparser->get_max_id();
+			$current = !empty($this->resume_data['range-max']) ? $this->resume_data['range-max'] : $reparser->get_max_id();
-			$size = !empty($this->resume_data[$this->reparser_name]['range-size']) ? $this->resume_data[$this->reparser_name]['range-size'] : self::SIZE;
+			$size = !empty($this->resume_data['range-size']) ? $this->resume_data['range-size'] : self::SIZE;
 
 			if ($current >= $min)
 			{
@@ -153,45 +158,11 @@ class reparser extends \phpbb\cron\task\base
 
 				$reparser->reparse_range($start, $end);
 
-				$this->update_resume_data($this->reparser_name, $min, $start - 1, $size);
+				$this->reparser_manager->update_resume_data($this->reparser_name, $min, $start - 1, $size);
 			}
 
+			$this->config->set($this->reparser_name . '_last_cron', time());
 			$this->reparse_lock->release();
 		}
 	}
-
-	/**
-	 * Load the resume data from the database
-	 */
-	protected function load_resume_data()
-	{
-		$resume_data = $this->config_text->get('reparser_resume');
-		$this->resume_data = (empty($resume_data)) ? array() : unserialize($resume_data);
-	}
-
-	/**
-	 * Save the resume data to the database
-	 */
-	protected function save_resume_data()
-	{
-		$this->config_text->set('reparser_resume', serialize($this->resume_data));
-	}
-
-	/**
-	 * Save the resume data to the database
-	 *
-	 * @param string	$name		Reparser name
-	 * @param int		$min		Lowest record ID
-	 * @param int		$current	Current ID
-	 * @param int		$size		Number of records to process at a time
-	 */
-	protected function update_resume_data($name, $min, $current, $size)
-	{
-		$this->resume_data[$name] = array(
-			'range-min'  => $min,
-			'range-max'  => $current,
-			'range-size' => $size,
-		);
-		$this->save_resume_data();
-	}
 }

phpBB/phpbb/db/migration/data/v320/text_reparser.php

@@ -13,7 +13,7 @@
 
 namespace phpbb\db\migration\data\v320;
 
-class reparse_fast extends \phpbb\db\migration\container_aware_migration
+class text_reparser extends \phpbb\db\migration\container_aware_migration
 {
 	static public function depends_on()
 	{
@@ -29,6 +29,16 @@ class reparse_fast extends \phpbb\db\migration\container_aware_migration
 	{
 		return array(
 			array('config.add', array('reparse_lock', 0, true)),
+			array('config.add', array('text_reparser.pm_text_cron_interval', 0)),
+			array('config.add', array('text_reparser.pm_text_last_cron', 0)),
+			array('config.add', array('text_reparser.poll_option_cron_interval', 0)),
+			array('config.add', array('text_reparser.poll_option_last_cron', 0)),
+			array('config.add', array('text_reparser.poll_title_cron_interval', 0)),
+			array('config.add', array('text_reparser.poll_title_last_cron', 0)),
+			array('config.add', array('text_reparser.post_text_cron_interval', 0)),
+			array('config.add', array('text_reparser.post_text_last_cron', 0)),
+			array('config.add', array('text_reparser.user_signature_cron_interval', 0)),
+			array('config.add', array('text_reparser.user_signature_last_cron', 0)),
 			array('custom', array(array($this, 'reparse'))),
 		);
 	}
@@ -36,7 +46,7 @@ class reparse_fast extends \phpbb\db\migration\container_aware_migration
 	public function reparse($resume_data)
 	{
 		// Somtimes a cron job is too much
-		$limit = 200;
+		$limit = 100;
 		$fast_reparsers = array(
 			'text_reparser.contact_admin_info',
 			'text_reparser.forum_description',
@@ -73,6 +83,10 @@ class reparse_fast extends \phpbb\db\migration\container_aware_migration
 
 			if ($start === 1)
 			{
+				// Prevent CLI command from running these reparsers again
+				$reparser_manager = $this->container->get('text_reparser.manager');
+				$reparser_manager->update_resume_data($fast_reparsers[$resume_data['reparser']], 1, 0, $limit);
+
 				$resume_data['reparser']++;
 			}
 		}

phpBB/phpbb/textreparser/manager.php

@@ -0,0 +1,128 @@
+<?php
+/**
+ *
+ * This file is part of the phpBB Forum Software package.
+ *
+ * @copyright (c) phpBB Limited <https://www.phpbb.com>
+ * @license GNU General Public License, version 2 (GPL-2.0)
+ *
+ * For full copyright and license information, please see
+ * the docs/CREDITS.txt file.
+ *
+ */
+
+namespace phpbb\textreparser;
+
+class manager
+{
+	/**
+	 * @var \phpbb\config\config
+	 */
+	protected $config;
+
+	/**
+	 * @var \phpbb\config\db_text
+	 */
+	protected $config_text;
+
+	/**
+	 * @var \phpbb\di\service_collection
+	 */
+	protected $reparsers;
+
+	/**
+	 * @var array
+	 */
+	protected $resume_data;
+
+	/**
+	 * Constructor
+	 *
+	 * @param \phpbb\config\config			$config
+	 * @param \phpbb\config\db_text			$config_text
+	 * @param \phpbb\di\service_collection	$reparsers
+	 */
+	public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\di\service_collection $reparsers)
+	{
+		$this->config = $config;
+		$this->config_text = $config_text;
+		$this->reparsers = $reparsers;
+	}
+
+	/**
+	 * Loads resume data from the database
+	 *
+	 * @param string	$name	Name of the reparser to which the resume data belongs
+	 *
+	 * @return array
+	 */
+	public function get_resume_data($name)
+	{
+		if ($this->resume_data === null)
+		{
+			$resume_data = $this->config_text->get('reparser_resume');
+			$this->resume_data = !empty($resume_data) ? unserialize($resume_data) : array();
+		}
+
+		return isset($this->resume_data[$name]) ? $this->resume_data[$name] : array();
+	}
+
+	/**
+	 * Updates the resume data in the database
+	 *
+	 * @param string	$name		Name of the reparser to which the resume data belongs
+	 * @param int		$min		Lowest record ID
+	 * @param int		$current	Current record ID
+	 * @param int		$size		Number of records to process at a time
+	 * @param bool		$update_db	True if the resume data should be written to the database, false if not. (default: true)
+	 */
+	public function update_resume_data($name, $min, $current, $size, $update_db = true)
+	{
+		// Prevent overwriting the old, stored array
+		if ($this->resume_data === null)
+		{
+			$this->get_resume_data('');
+		}
+
+		$this->resume_data[$name] = array(
+			'range-min'		=> $min,
+			'range-max'		=> $current,
+			'range-size'	=> $size,
+		);
+
+		if ($update_db)
+		{
+			$this->config_text->set('reparser_resume', serialize($this->resume_data));
+		}
+	}
+
+	/**
+	 * Sets the interval for a text_reparser cron task
+	 *
+	 * @param string	$name		Name of the reparser to schedule
+	 * @param int		$interval	Interval in seconds, -1 to disable the cron task
+	 */
+	public function schedule($name, $interval)
+	{
+		if (isset($this->reparsers[$name]) && isset($this->config[$name . '_cron_interval']))
+		{
+			$this->config->set($name . '_cron_interval', $interval);
+		}
+	}
+
+	/**
+	 * Sets the interval for all text_reparser cron tasks
+	 *
+	 * @param int	$interval	Interval in seconds, -1 to disable the cron task
+	 */
+	public function schedule_all($interval)
+	{
+		// This way we don't construct every registered reparser
+		$reparser_array = array_keys($this->reparsers->getArrayCopy());
+
+		foreach ($reparser_array as $reparser)
+		{
+			$this->schedule($reparser, $interval);
+		}
+	}
+}