mirror of
https://github.com/ezyang/htmlpurifier.git
synced 2025-07-30 19:00:10 +02:00
Improve auto-paragraph to preserve newlines and handle edge-cases better.
This is a very large commit that includes numerous improvements to the AutoParagraph injector. These are: * Rewritten flow control of the injector to use almost exclusively binary conditionals. * Improved inline documentation with "State" comments, which give concise examples of what the token stack looks like at flow points. * Documentation for all flow branches, even those with no actions. * Factoring out of common operations to improve readability, especially the new iterator private methods. * Expanded test-suite which covers new flow points, and corrects some errors in previous cases. Signed-off-by: Edward Z. Yang <edwardzyang@thewritingpot.com>
This commit is contained in:
Edward Z. Yang
@@ -3,6 +3,8 @@
|
||||
/**
|
||||
* Injector that auto paragraphs text in the root node based on
|
||||
* double-spacing.
|
||||
* @todo Ensure all states are unit tested, including variations as well.
|
||||
* @todo Make a graph of the flow control for this Injector.
|
||||
*/
|
||||
class HTMLPurifier_Injector_AutoParagraph extends HTMLPurifier_Injector
|
||||
{
|
||||
@@ -18,116 +20,177 @@ class HTMLPurifier_Injector_AutoParagraph extends HTMLPurifier_Injector
|
||||
|
||||
public function handleText(&$token) {
|
||||
$text = $token->data;
|
||||
if (empty($this->currentNesting)) {
|
||||
if (!$this->allowsElement('p')) return;
|
||||
// case 1: we're in root node (and it allows paragraphs)
|
||||
$token = array($this->_pStart());
|
||||
$this->_splitText($text, $token);
|
||||
} elseif ($this->currentNesting[count($this->currentNesting)-1]->name == 'p') {
|
||||
// case 2: we're in a paragraph
|
||||
$token = array();
|
||||
$this->_splitText($text, $token);
|
||||
} elseif ($this->allowsElement('p')) {
|
||||
// case 3: we're in an element that allows paragraphs
|
||||
if (strpos($text, "\n\n") !== false) {
|
||||
// case 3.1: this text node has a double-newline
|
||||
$token = array($this->_pStart());
|
||||
$this->_splitText($text, $token);
|
||||
} else {
|
||||
$ok = false;
|
||||
// test if up-coming tokens are either block or have
|
||||
// a double newline in them
|
||||
$nesting = 0;
|
||||
for ($i = $this->inputIndex + 1; isset($this->inputTokens[$i]); $i++) {
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Start){
|
||||
if (!$this->_isInline($this->inputTokens[$i])) {
|
||||
// we haven't found a double-newline, and
|
||||
// we've hit a block element, so don't paragraph
|
||||
$ok = false;
|
||||
break;
|
||||
}
|
||||
$nesting++;
|
||||
}
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_End) {
|
||||
if ($nesting <= 0) break;
|
||||
$nesting--;
|
||||
}
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Text) {
|
||||
// found it!
|
||||
if (strpos($this->inputTokens[$i]->data, "\n\n") !== false) {
|
||||
$ok = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Does the current parent allow <p> tags?
|
||||
if ($this->allowsElement('p')) {
|
||||
if (empty($this->currentNesting) || strpos($text, "\n\n") !== false) {
|
||||
// Note that we have differing behavior when dealing with text
|
||||
// in the anonymous root node, or a node inside the document.
|
||||
// If the text as a double-newline, the treatment is the same;
|
||||
// if it doesn't, see the next if-block if you're in the document.
|
||||
|
||||
$i = $nesting = null;
|
||||
if (!$this->_forwardUntilEndToken($i, $current, $nesting) && $token->is_whitespace) {
|
||||
// State 1.1: ... ^ (whitespace, then document end)
|
||||
// ----
|
||||
// This is a degenerate case
|
||||
} else {
|
||||
// State 1.2: PAR1
|
||||
// ----
|
||||
|
||||
// State 1.3: PAR1\n\nPAR2
|
||||
// ------------
|
||||
|
||||
// State 1.4: <div>PAR1\n\nPAR2 (see State 2)
|
||||
// ------------
|
||||
$token = array($this->_pStart());
|
||||
$this->_splitText($text, $token);
|
||||
}
|
||||
if ($ok) {
|
||||
// case 3.2: this text node is next to another node
|
||||
// that will start a paragraph
|
||||
} else {
|
||||
// State 2: <div>PAR1... (similar to 1.4)
|
||||
// ----
|
||||
|
||||
// We're in an element that allows paragraph tags, but we're not
|
||||
// sure if we're going to need them.
|
||||
if ($this->_pLookAhead()) {
|
||||
// State 2.1: <div>PAR1<b>PAR1\n\nPAR2
|
||||
// ----
|
||||
// Note: This will always be the first child, since any
|
||||
// previous inline element would have triggered this very
|
||||
// same routine, and found the double newline. One possible
|
||||
// exception would be a comment.
|
||||
$token = array($this->_pStart(), $token);
|
||||
} else {
|
||||
// State 2.2.1: <div>PAR1<div>
|
||||
// ----
|
||||
|
||||
// State 2.2.2: <div>PAR1<b>PAR1</b></div>
|
||||
// ----
|
||||
}
|
||||
}
|
||||
// Is the current parent a <p> tag?
|
||||
} elseif (
|
||||
!empty($this->currentNesting) &&
|
||||
$this->currentNesting[count($this->currentNesting)-1]->name == 'p'
|
||||
) {
|
||||
// State 3.1: ...<p>PAR1
|
||||
// ----
|
||||
|
||||
// State 3.2: ...<p>PAR1\n\nPAR2
|
||||
// ------------
|
||||
$token = array();
|
||||
$this->_splitText($text, $token);
|
||||
// Abort!
|
||||
} else {
|
||||
// State 4.1: ...<b>PAR1
|
||||
// ----
|
||||
|
||||
// State 4.2: ...<b>PAR1\n\nPAR2
|
||||
// ------------
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public function handleElement(&$token) {
|
||||
// check if we're inside a tag already
|
||||
if (!empty($this->currentNesting)) {
|
||||
if ($this->allowsElement('p')) {
|
||||
// special case: we're in an element that allows paragraphs
|
||||
|
||||
// this token is already paragraph, abort
|
||||
if ($token->name == 'p') return;
|
||||
|
||||
// this token is a block level, abort
|
||||
if (!$this->_isInline($token)) return;
|
||||
|
||||
// check if this token is adjacent to the parent token
|
||||
$prev = $this->inputTokens[$this->inputIndex - 1];
|
||||
if (!$prev instanceof HTMLPurifier_Token_Start) {
|
||||
// not adjacent, we can abort early
|
||||
// add lead paragraph tag if our token is inline
|
||||
// and the previous tag was an end paragraph
|
||||
if (
|
||||
$prev->name == 'p' && $prev instanceof HTMLPurifier_Token_End &&
|
||||
$this->_isInline($token)
|
||||
) {
|
||||
$token = array($this->_pStart(), $token);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// this token is the first child of the element that allows
|
||||
// paragraph. We have to peek ahead and see whether or not
|
||||
// there is anything inside that suggests that a paragraph
|
||||
// will be needed
|
||||
$ok = false;
|
||||
// maintain a mini-nesting counter, this lets us bail out
|
||||
// early if possible
|
||||
$j = 1; // current nesting, one is due to parent (we recalculate current token)
|
||||
for ($i = $this->inputIndex; isset($this->inputTokens[$i]); $i++) {
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Start) $j++;
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_End) $j--;
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Text) {
|
||||
if (strpos($this->inputTokens[$i]->data, "\n\n") !== false) {
|
||||
$ok = true;
|
||||
break;
|
||||
// We don't have to check if we're already in a <p> tag for block
|
||||
// tokens, because the tag would have been autoclosed by MakeWellFormed.
|
||||
if ($this->allowsElement('p')) {
|
||||
if (!empty($this->currentNesting)) {
|
||||
if ($this->_isInline($token)) {
|
||||
// State 1: <div>...<b>
|
||||
// ---
|
||||
|
||||
// Check if this token is adjacent to the parent token
|
||||
// (seek backwards until token isn't whitespace)
|
||||
$i = null;
|
||||
$this->_backward($i, $prev);
|
||||
|
||||
if (!$prev instanceof HTMLPurifier_Token_Start) {
|
||||
// Token wasn't adjacent
|
||||
|
||||
if (
|
||||
$prev instanceof HTMLPurifier_Token_Text &&
|
||||
substr($prev->data, -2) === "\n\n"
|
||||
) {
|
||||
// State 1.1.4: <div><p>PAR1</p>\n\n<b>
|
||||
// ---
|
||||
|
||||
// Quite frankly, this should be handled by splitText
|
||||
$token = array($this->_pStart(), $token);
|
||||
} else {
|
||||
// State 1.1.1: <div><p>PAR1</p><b>
|
||||
// ---
|
||||
|
||||
// State 1.1.2: <div><br /><b>
|
||||
// ---
|
||||
|
||||
// State 1.1.3: <div>PAR<b>
|
||||
// ---
|
||||
}
|
||||
|
||||
} else {
|
||||
// State 1.2.1: <div><b>
|
||||
// ---
|
||||
|
||||
// Lookahead to see if <p> is needed.
|
||||
if ($this->_pLookAhead()) {
|
||||
// State 1.3.1: <div><b>PAR1\n\nPAR2
|
||||
// ---
|
||||
$token = array($this->_pStart(), $token);
|
||||
} else {
|
||||
// State 1.3.2: <div><b>PAR1</b></div>
|
||||
// ---
|
||||
|
||||
// State 1.3.3: <div><b>PAR1</b><div></div>\n\n</div>
|
||||
// ---
|
||||
}
|
||||
}
|
||||
if ($j <= 0) break;
|
||||
} else {
|
||||
// State 2.3: ...<div>
|
||||
// -----
|
||||
}
|
||||
if ($ok) {
|
||||
} else {
|
||||
if ($this->_isInline($token)) {
|
||||
// State 3.1: <b>
|
||||
// ---
|
||||
// This is where the {p} tag is inserted, not reflected in
|
||||
// inputTokens yet, however.
|
||||
$token = array($this->_pStart(), $token);
|
||||
} else {
|
||||
// State 3.2: <div>
|
||||
// -----
|
||||
}
|
||||
|
||||
$i = null;
|
||||
if ($this->_backward($i, $prev)) {
|
||||
if (
|
||||
!$prev instanceof HTMLPurifier_Token_Text
|
||||
) {
|
||||
// State 3.1.1: ...</p>{p}<b>
|
||||
// ---
|
||||
|
||||
// State 3.2.1: ...</p><div>
|
||||
// -----
|
||||
|
||||
if (!is_array($token)) $token = array($token);
|
||||
array_unshift($token, new HTMLPurifier_Token_Text("\n\n"));
|
||||
} else {
|
||||
// State 3.1.2: ...</p>\n\n{p}<b>
|
||||
// ---
|
||||
|
||||
// State 3.2.2: ...</p>\n\n<div>
|
||||
// -----
|
||||
|
||||
// Note: PAR<ELEM> cannot occur because PAR would have been
|
||||
// wrapped in <p> tags.
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
// State 2.2: <ul><li>
|
||||
// ----
|
||||
|
||||
// State 2.4: <p><b>
|
||||
// ---
|
||||
}
|
||||
|
||||
// check if the start tag counts as a "block" element
|
||||
if (!$this->_isInline($token)) return;
|
||||
|
||||
// append a paragraph tag before the token
|
||||
$token = array($this->_pStart(), $token);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,96 +205,80 @@ class HTMLPurifier_Injector_AutoParagraph extends HTMLPurifier_Injector
|
||||
*/
|
||||
private function _splitText($data, &$result) {
|
||||
$raw_paragraphs = explode("\n\n", $data);
|
||||
|
||||
// remove empty paragraphs
|
||||
$paragraphs = array();
|
||||
$paragraphs = array(); // without empty paragraphs
|
||||
$needs_start = false;
|
||||
$needs_end = false;
|
||||
|
||||
$c = count($raw_paragraphs);
|
||||
if ($c == 1) {
|
||||
// there were no double-newlines, abort quickly
|
||||
// There were no double-newlines, abort quickly. In theory this
|
||||
// should never happen.
|
||||
$result[] = new HTMLPurifier_Token_Text($data);
|
||||
return;
|
||||
}
|
||||
|
||||
for ($i = 0; $i < $c; $i++) {
|
||||
$par = $raw_paragraphs[$i];
|
||||
if (trim($par) !== '') {
|
||||
$paragraphs[] = $par;
|
||||
continue;
|
||||
}
|
||||
if ($i == 0 && empty($result)) {
|
||||
// The empty result indicates that the AutoParagraph
|
||||
// injector did not add any start paragraph tokens.
|
||||
// The fact that the first paragraph is empty indicates
|
||||
// that there was a double-newline at the start of the
|
||||
// data.
|
||||
// Combined together, this means that we are in a paragraph,
|
||||
// and the newline means we should start a new one.
|
||||
$result[] = new HTMLPurifier_Token_End('p');
|
||||
// However, the start token should only be added if
|
||||
// there is more processing to be done (i.e. there are
|
||||
// real paragraphs in here). If there are none, the
|
||||
// next start paragraph tag will be handled by the
|
||||
// next run-around the injector
|
||||
$needs_start = true;
|
||||
} elseif ($i + 1 == $c) {
|
||||
// a double-paragraph at the end indicates that
|
||||
// there is an overriding need to start a new paragraph
|
||||
// for the next section. This has no effect until
|
||||
// we've processed all of the other paragraphs though
|
||||
$needs_end = true;
|
||||
} else {
|
||||
if ($i == 0) {
|
||||
// Double newline at the front
|
||||
if (empty($result)) {
|
||||
// The empty result indicates that the AutoParagraph
|
||||
// injector did not add any start paragraph tokens.
|
||||
// This means that we have been in a paragraph for
|
||||
// a while, and the newline means we should start a new one.
|
||||
$result[] = new HTMLPurifier_Token_End('p');
|
||||
$result[] = new HTMLPurifier_Token_Text("\n\n");
|
||||
// However, the start token should only be added if
|
||||
// there is more processing to be done (i.e. there are
|
||||
// real paragraphs in here). If there are none, the
|
||||
// next start paragraph tag will be handled by the
|
||||
// next call to the injector
|
||||
$needs_start = true;
|
||||
} else {
|
||||
// We just started a new paragraph!
|
||||
// Reinstate a double-newline for presentation's sake, since
|
||||
// it was in the source code.
|
||||
array_unshift($result, new HTMLPurifier_Token_Text("\n\n"));
|
||||
}
|
||||
} elseif ($i + 1 == $c) {
|
||||
// Double newline at the end
|
||||
// There should be a trailing </p> when we're finally done.
|
||||
$needs_end = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if there are no "real" paragraphs to be processed
|
||||
// Check if this was just a giant blob of whitespace. Move this earlier,
|
||||
// perhaps?
|
||||
if (empty($paragraphs)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// add a start tag if an end tag was added while processing
|
||||
// the raw paragraphs (that happens if there's a leading double
|
||||
// newline)
|
||||
if ($needs_start) $result[] = $this->_pStart();
|
||||
|
||||
// append the paragraphs onto the result
|
||||
foreach ($paragraphs as $par) {
|
||||
$result[] = new HTMLPurifier_Token_Text($par);
|
||||
$result[] = new HTMLPurifier_Token_End('p');
|
||||
// Add the start tag indicated by \n\n at the beginning of $data
|
||||
if ($needs_start) {
|
||||
$result[] = $this->_pStart();
|
||||
}
|
||||
|
||||
// remove trailing start token, if one is needed, it will
|
||||
// be handled the next time this injector is called
|
||||
array_pop($result);
|
||||
|
||||
// check the outside to determine whether or not the
|
||||
// end paragraph tag should be removed. It should be removed
|
||||
// unless the next non-whitespace token is a paragraph
|
||||
// or a block element.
|
||||
$remove_paragraph_end = true;
|
||||
|
||||
if (!$needs_end) {
|
||||
// Start of the checks one after the current token's index
|
||||
for ($i = $this->inputIndex + 1; isset($this->inputTokens[$i]); $i++) {
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Start || $this->inputTokens[$i] instanceof HTMLPurifier_Token_Empty) {
|
||||
$remove_paragraph_end = $this->_isInline($this->inputTokens[$i]);
|
||||
}
|
||||
// check if we can abort early (whitespace means we carry-on!)
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_Text && !$this->inputTokens[$i]->is_whitespace) break;
|
||||
// end tags will automatically be handled by MakeWellFormed,
|
||||
// so we don't have to worry about them
|
||||
if ($this->inputTokens[$i] instanceof HTMLPurifier_Token_End) break;
|
||||
}
|
||||
} else {
|
||||
$remove_paragraph_end = false;
|
||||
// Append the paragraphs onto the result
|
||||
foreach ($paragraphs as $par) {
|
||||
$result[] = new HTMLPurifier_Token_Text($par);
|
||||
$result[] = new HTMLPurifier_Token_End('p');
|
||||
$result[] = new HTMLPurifier_Token_Text("\n\n");
|
||||
$result[] = $this->_pStart();
|
||||
}
|
||||
|
||||
// check the outside to determine whether or not the
|
||||
// end paragraph tag should be removed
|
||||
if ($remove_paragraph_end) {
|
||||
array_pop($result);
|
||||
// Remove trailing start token; Injector will handle this later if
|
||||
// it was indeed needed. This prevents from needing to do a lookahead,
|
||||
// at the cost of a lookbehind later.
|
||||
array_pop($result);
|
||||
|
||||
// If there is no need for an end tag, remove all of it and let
|
||||
// MakeWellFormed close it later.
|
||||
if (!$needs_end) {
|
||||
array_pop($result); // removes \n\n
|
||||
array_pop($result); // removes </p>
|
||||
}
|
||||
|
||||
}
|
||||
@@ -244,5 +291,112 @@ class HTMLPurifier_Injector_AutoParagraph extends HTMLPurifier_Injector
|
||||
return isset($this->htmlDefinition->info['p']->child->elements[$token->name]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks ahead in the token list and determines whether or not we need
|
||||
* to insert a <p> tag.
|
||||
*/
|
||||
private function _pLookAhead() {
|
||||
$this->_current($i, $current);
|
||||
if ($current instanceof HTMLPurifier_Token_Start) $nesting = 1;
|
||||
else $nesting = 0;
|
||||
$ok = false;
|
||||
while ($this->_forwardUntilEndToken($i, $current, $nesting)) {
|
||||
$result = $this->_checkNeedsP($current);
|
||||
if ($result !== null) {
|
||||
$ok = $result;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return $ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterator function, which starts with the next token and continues until
|
||||
* you reach the end of the input tokens.
|
||||
* @warning Please prevent previous references from interfering with this
|
||||
* functions by setting $i = null beforehand!
|
||||
* @param &$i Current integer index variable for inputTokens
|
||||
* @param &$current Current token variable. Do NOT use $token, as that variable is also a reference
|
||||
*/
|
||||
private function _forward(&$i, &$current) {
|
||||
if ($i === null) $i = $this->inputIndex + 1;
|
||||
else $i++;
|
||||
if (!isset($this->inputTokens[$i])) return false;
|
||||
$current = $this->inputTokens[$i];
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Similar to _forward, but accepts a third parameter $nesting (which
|
||||
* should be initialized at 0) and stops when we hit the end tag
|
||||
* for the node $this->inputIndex starts in.
|
||||
*/
|
||||
private function _forwardUntilEndToken(&$i, &$current, &$nesting) {
|
||||
$result = $this->_forward($i, $current);
|
||||
if (!$result) return false;
|
||||
if ($nesting === null) $nesting = 0;
|
||||
if ($current instanceof HTMLPurifier_Token_Start) $nesting++;
|
||||
elseif ($current instanceof HTMLPurifier_Token_End) {
|
||||
if ($nesting <= 0) return false;
|
||||
$nesting--;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterator function, starts with the previous token and continues until
|
||||
* you reach the beginning of input tokens.
|
||||
* @warning Please prevent previous references from interfering with this
|
||||
* functions by setting $i = null beforehand!
|
||||
* @param &$i Current integer index variable for inputTokens
|
||||
* @param &$current Current token variable. Do NOT use $token, as that variable is also a reference
|
||||
*/
|
||||
private function _backward(&$i, &$current) {
|
||||
if ($i === null) $i = $this->inputIndex - 1;
|
||||
else $i--;
|
||||
if ($i < 0) return false;
|
||||
$current = $this->inputTokens[$i];
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the iterator at the current position. Use in a do {} while;
|
||||
* loop to force the _forward and _backward functions to start at the
|
||||
* current location.
|
||||
* @warning Please prevent previous references from interfering with this
|
||||
* functions by setting $i = null beforehand!
|
||||
* @param &$i Current integer index variable for inputTokens
|
||||
* @param &$current Current token variable. Do NOT use $token, as that variable is also a reference
|
||||
*/
|
||||
private function _current(&$i, &$current) {
|
||||
if ($i === null) $i = $this->inputIndex;
|
||||
$current = $this->inputTokens[$i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if a particular token requires an earlier inline token
|
||||
* to get a paragraph. This should be used with _forwardUntilEndToken
|
||||
*/
|
||||
private function _checkNeedsP($current) {
|
||||
if ($current instanceof HTMLPurifier_Token_Start){
|
||||
if (!$this->_isInline($current)) {
|
||||
// <div>PAR1<div>
|
||||
// ----
|
||||
// Terminate early, since we hit a block element
|
||||
return false;
|
||||
}
|
||||
} elseif ($current instanceof HTMLPurifier_Token_Text) {
|
||||
if (strpos($current->data, "\n\n") !== false) {
|
||||
// <div>PAR1<b>PAR1\n\nPAR2
|
||||
// ----
|
||||
return true;
|
||||
} else {
|
||||
// <div>PAR1<b>PAR1...
|
||||
// ----
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user