Merged 438:439, 440:441, and 442:457 from trunk/ to branches/1.1/, mostly major work done for 1.1.1 release.

- Various documentation updates
- Fixed fatal error in benchmark scripts, slightly augmented
- As far as possible, whitespace is preserved in-between table children
- Configuration option to optionally Tidy up output for indentation to make up for dropped whitespace by DOMLex (pretty-printing for the entire application should be done by a page-wide Tidy)
- Sample test-settings.php file included

Unrelated unmerged edit: removed irrelevant 1.2.0 release notes, those only exist in the trunk.

git-svn-id: http://htmlpurifier.org/svnroot/htmlpurifier/branches/1.1@458 48356398-32a2-884e-a903-53898d9a118a

library/HTMLPurifier/AttrDef/Host.php

@@ -11,9 +11,14 @@ class HTMLPurifier_AttrDef_Host extends HTMLPurifier_AttrDef
 {
     
     /**
-     * Instances of HTMLPurifier_AttrDef_IPv4 and HTMLPurifier_AttrDef_IPv6
+     * Instance of HTMLPurifier_AttrDef_IPv4 sub-validator
      */
-    var $ipv4, $ipv6;
+    var $ipv4;
+    
+    /**
+     * Instance of HTMLPurifier_AttrDef_IPv6 sub-validator
+     */
+    var $ipv6;
     
     function HTMLPurifier_AttrDef_Host() {
         $this->ipv4 = new HTMLPurifier_AttrDef_IPv4();

library/HTMLPurifier/ChildDef.php

@@ -327,6 +327,8 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
         $is_collecting = false; // are we globbing together tokens to package
                                 // into one of the collectors?
         $collection = array(); // collected nodes
+        $tag_index = 0; // the first node might be whitespace,
+                            // so this tells us where the start tag is
         
         foreach ($tokens_of_children as $token) {
             $is_child = ($nesting == 0);
@@ -344,7 +346,7 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
                 if ($is_child) {
                     // okay, let's stash the tokens away
                     // first token tells us the type of the collection
-                    switch ($collection[0]->name) {
+                    switch ($collection[$tag_index]->name) {
                         case 'tr':
                         case 'tbody':
                             $content[] = $collection;
@@ -356,13 +358,13 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
                         case 'thead':
                         case 'tfoot':
                             // access the appropriate variable, $thead or $tfoot
-                            $var = $collection[0]->name;
+                            $var = $collection[$tag_index]->name;
                             if ($$var === false) {
                                 $$var = $collection;
                             } else {
                                 // transmutate the first and less entries into
                                 // tbody tags, and then put into content
-                                $collection[0]->name = 'tbody';
+                                $collection[$tag_index]->name = 'tbody';
                                 $collection[count($collection)-1]->name = 'tbody';
                                 $content[] = $collection;
                             }
@@ -373,6 +375,7 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
                     }
                     $collection = array();
                     $is_collecting = false;
+                    $tag_index = 0;
                 } else {
                     // add the node to the collection
                     $collection[] = $token;
@@ -387,7 +390,9 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
                 if ($token->name == 'col') {
                     // the only empty tag in the possie, we can handle it
                     // immediately
-                    $cols[] = array($token);
+                    $cols[] = array_merge($collection, array($token));
+                    $collection = array();
+                    $tag_index = 0;
                     continue;
                 }
                 switch($token->name) {
@@ -401,7 +406,10 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
                         $collection[] = $token;
                         continue;
                     default:
-                        // unrecognized, drop silently
+                        if ($token->type == 'text' && $token->is_whitespace) {
+                            $collection[] = $token;
+                            $tag_index++;
+                        }
                         continue;
                 }
             }
@@ -415,6 +423,10 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
         if ($thead !== false)   $ret = array_merge($ret, $thead);
         if ($tfoot !== false)   $ret = array_merge($ret, $tfoot);
         foreach ($content as $token_array) $ret = array_merge($ret, $token_array);
+        if (!empty($collection) && $is_collecting == false){
+            // grab the trailing space
+            $ret = array_merge($ret, $collection);
+        }
         
         array_pop($tokens_of_children); // remove phantom token
         
@@ -423,4 +435,4 @@ class HTMLPurifier_ChildDef_Table extends HTMLPurifier_ChildDef
     }
 }
 
-?>
+?>

library/HTMLPurifier/Encoder.php

@@ -88,7 +88,7 @@ class HTMLPurifier_Encoder
         if ($iconv && !$force_php) {
             // do the shortcut way
             $str = @iconv('UTF-8', 'UTF-8//IGNORE', $str);
-            return strtr($str, $non_sgml_chars);;
+            return strtr($str, $non_sgml_chars);
         }
         
         $mState = 0; // cached expected number of octets after the current octet

library/HTMLPurifier/Generator.php

@@ -23,6 +23,21 @@ HTMLPurifier_ConfigSchema::define(
     'This directive was available since 1.1.'
 );
 
+// extension constraints could be factored into ConfigSchema
+HTMLPurifier_ConfigSchema::define(
+    'Core', 'TidyFormat', false, 'bool',
+    '<p>Determines whether or not to run Tidy on the final output for pretty '.
+    'formatting reasons, such as indentation and wrap.</p><p>This can greatly '.
+    'improve readability for editors who are hand-editing the HTML, but is '.
+    'by no means necessary as HTML Purifier has already fixed all major '.
+    'errors the HTML may have had. Tidy is a non-default extension, and this directive '.
+    'will silently fail if Tidy is not available.</p><p>If you are looking to make '.
+    'the overall look of your page\'s source better, I recommend running Tidy '.
+    'on the entire page rather than just user-content (after all, the '.
+    'indentation relative to the containing blocks will be incorrect).</p><p>This '.
+    'directive was available since 1.1.1.</p>'
+);
+
 /**
  * Generates HTML from tokens.
  */
@@ -56,6 +71,30 @@ class HTMLPurifier_Generator
         foreach ($tokens as $token) {
             $html .= $this->generateFromToken($token);
         }
+        if ($config->get('Core', 'TidyFormat') && extension_loaded('tidy')) {
+            
+            $tidy_options = array(
+               'indent'=> true,
+               'output-xhtml' => $this->_xhtml,
+               'show-body-only' => true,
+               'indent-spaces' => 2,
+               'wrap' => 68,
+            );
+            if (version_compare(PHP_VERSION, '5', '<')) {
+                tidy_set_encoding('utf8');
+                foreach ($tidy_options as $key => $value) {
+                    tidy_setopt($key, $value);
+                }
+                tidy_parse_string($html);
+                tidy_clean_repair();
+                $html = tidy_get_output();
+            } else {
+                $tidy = new Tidy;
+                $tidy->parseString($html, $tidy_options, 'utf8');
+                $tidy->cleanRepair();
+                $html = (string) $tidy;
+            }
+        }
         return $html;
     }
     

library/HTMLPurifier/Lexer/DOMLex.php

@@ -18,6 +18,13 @@ require_once 'HTMLPurifier/TokenFactory.php';
  * 
  * @note PHP's DOM extension does not actually parse any entities, we use
  *       our own function to do that.
+ * 
+ * @warning DOM tends to drop whitespace, which may wreak havoc on indenting.
+ *          If this is a huge problem, due to the fact that HTML is hand
+ *          edited and youa re unable to get a parser cache that caches the
+ *          the output of HTML Purifier while keeping the original HTML lying
+ *          around, you may want to run Tidy on the resulting output or use
+ *          HTMLPurifier_DirectLex
  */
 
 class HTMLPurifier_Lexer_DOMLex extends HTMLPurifier_Lexer