Merge pull request #563 from luizbills/master

bump version

.travis.yml

@@ -20,8 +20,10 @@ matrix:
     - php: nightly
     - php: hhvm-nightly
 
-before_script:
+install:
   - composer install --prefer-dist --no-interaction --no-progress
 
 script:
   - vendor/bin/phpunit
+  - vendor/bin/phpunit test/CommonMarkTestWeak.php || true
+  - '[ -z "$TRAVIS_TAG" ] || [ "$TRAVIS_TAG" == "$(php -r "require(\"Parsedown.php\"); echo Parsedown::version;")" ]'

Parsedown.php

@@ -17,7 +17,7 @@ class Parsedown
 {
     # ~
 
-    const version = '1.6.0';
+    const version = '1.7.1';
 
     # ~
 
@@ -420,7 +420,7 @@ class Parsedown
 
     protected function blockFencedCode($Line)
     {
-        if (preg_match('/^['.$Line['text'][0].']{3,}[ ]*([\w-]+)?[ ]*$/', $Line['text'], $matches))
+        if (preg_match('/^['.$Line['text'][0].']{3,}[ ]*([^`]+)?[ ]*$/', $Line['text'], $matches))
         {
             $Element = array(
                 'name' => 'code',
@@ -569,6 +569,8 @@ class Parsedown
             {
                 $Block['li']['text'] []= '';
 
+                $Block['loose'] = true;
+
                 unset($Block['interrupted']);
             }
 
@@ -617,6 +619,22 @@ class Parsedown
         }
     }
 
+    protected function blockListComplete(array $Block)
+    {
+        if (isset($Block['loose']))
+        {
+            foreach ($Block['element']['text'] as &$li)
+            {
+                if (end($li['text']) !== '')
+                {
+                    $li['text'] []= '';
+                }
+            }
+        }
+
+        return $Block;
+    }
+
     #
     # Quote
 
@@ -1019,7 +1037,7 @@ class Parsedown
     # ~
     #
 
-    public function line($text)
+    public function line($text, $nonNestables=array())
     {
         $markup = '';
 
@@ -1035,6 +1053,13 @@ class Parsedown
 
             foreach ($this->InlineTypes[$marker] as $inlineType)
             {
+                # check to see if the current inline type is nestable in the current context
+
+                if ( ! empty($nonNestables) and in_array($inlineType, $nonNestables))
+                {
+                    continue;
+                }
+
                 $Inline = $this->{'inline'.$inlineType}($Excerpt);
 
                 if ( ! isset($Inline))
@@ -1056,6 +1081,13 @@ class Parsedown
                     $Inline['position'] = $markerPosition;
                 }
 
+                # cause the new element to 'inherit' our non nestables
+
+                foreach ($nonNestables as $non_nestable)
+                {
+                    $Inline['element']['nonNestables'][] = $non_nestable;
+                }
+
                 # the text that comes before the inline
                 $unmarkedText = substr($text, 0, $Inline['position']);
 
@@ -1214,6 +1246,7 @@ class Parsedown
         $Element = array(
             'name' => 'a',
             'handler' => 'line',
+            'nonNestables' => array('Url', 'Link'),
             'text' => null,
             'attributes' => array(
                 'href' => null,
@@ -1446,9 +1479,14 @@ class Parsedown
         {
             $markup .= '>';
 
+            if (!isset($Element['nonNestables'])) 
+            {
+                $Element['nonNestables'] = array();
+            }
+
             if (isset($Element['handler']))
             {
-                $markup .= $this->{$Element['handler']}($Element['text']);
+                $markup .= $this->{$Element['handler']}($Element['text'], $Element['nonNestables']);
             }
             else
             {

README.md

@@ -38,7 +38,32 @@ More examples in [the wiki](https://github.com/erusev/parsedown/wiki/) and in [t
 
 ### Security
 
-Parsedown does not sanitize the HTML that it generates. When you deal with untrusted content (ex: user comments) you should also use a HTML sanitizer like [HTML Purifier](http://htmlpurifier.org/).
+Parsedown is capable of escaping user-input within the HTML that it generates. Additionally Parsedown will apply sanitisation to additional scripting vectors (such as scripting link destinations) that are introduced by the markdown syntax itself.
+
+To tell Parsedown that it is processing untrusted user-input, use the following:
+```php
+$parsedown = new Parsedown;
+$parsedown->setSafeMode(true);
+```
+
+If instead, you wish to allow HTML within untrusted user-input, but still want output to be free from XSS it is recommended that you make use of a HTML sanitiser that allows HTML tags to be whitelisted, like [HTML Purifier](http://htmlpurifier.org/).
+
+In both cases you should strongly consider employing defence-in-depth measures, like [deploying a Content-Security-Policy](https://scotthelme.co.uk/content-security-policy-an-introduction/) (a browser security feature) so that your page is likely to be safe even if an attacker finds a vulnerability in one of the first lines of defence above.
+
+#### Security of Parsedown Extensions
+
+Safe mode does not necessarily yield safe results when using extensions to Parsedown. Extensions should be evaluated on their own to determine their specific safety against XSS.
+
+### Escaping HTML
+> ⚠️  **WARNING:** This method isn't safe from XSS!
+
+If you wish to escape HTML **in trusted input**, you can use the following:
+```php
+$parsedown = new Parsedown;
+$parsedown->setMarkupEscaped(true);
+```
+
+Beware that this still allows users to insert unsafe scripting vectors, such as links like `[xss](javascript:alert%281%29)`.
 
 ### Questions
 
@@ -54,7 +79,7 @@ It passes most of the CommonMark tests. Most of the tests that don't pass deal w
 
 **Who uses it?**
 
-[phpDocumentor](http://www.phpdoc.org/), [October CMS](http://octobercms.com/), [Bolt CMS](http://bolt.cm/), [Kirby CMS](http://getkirby.com/), [Grav CMS](http://getgrav.org/), [Statamic CMS](http://www.statamic.com/), [Herbie CMS](http://www.getherbie.org/), [RaspberryPi.org](http://www.raspberrypi.org/), [Symfony demo](https://github.com/symfony/symfony-demo) and [more](https://packagist.org/packages/erusev/parsedown/dependents).
+[Laravel Framework](https://laravel.com/), [Bolt CMS](http://bolt.cm/), [Grav CMS](http://getgrav.org/), [Herbie CMS](http://www.getherbie.org/), [Kirby CMS](http://getkirby.com/), [October CMS](http://octobercms.com/), [Pico CMS](http://picocms.org), [Statamic CMS](http://www.statamic.com/), [phpDocumentor](http://www.phpdoc.org/), [RaspberryPi.org](http://www.raspberrypi.org/), [Symfony demo](https://github.com/symfony/symfony-demo) and [more](https://packagist.org/packages/erusev/parsedown/dependents).
 
 **How can I help?**
 

composer.json

@@ -13,12 +13,21 @@
         }
     ],
     "require": {
-        "php": ">=5.3.0"
+        "php": ">=5.3.0",
+        "ext-mbstring": "*"
     },
     "require-dev": {
         "phpunit/phpunit": "^4.8.35"
     },
     "autoload": {
         "psr-0": {"Parsedown": ""}
+    },
+    "autoload-dev": {
+        "psr-0": {
+            "TestParsedown": "test/",
+            "ParsedownTest": "test/",
+            "CommonMarkTest": "test/",
+            "CommonMarkTestWeak": "test/"
+        }
     }
 }

phpunit.xml.dist

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<phpunit bootstrap="test/bootstrap.php" colors="true">
+<phpunit bootstrap="vendor/autoload.php" colors="true">
 	<testsuites>
 		<testsuite>
 			<file>test/ParsedownTest.php</file>

test/CommonMarkTest.php

@@ -1,77 +0,0 @@
-<?php
-
-/**
- * Test Parsedown against the CommonMark spec.
- *
- * Some code based on the original JavaScript test runner by jgm.
- *
- * @link http://commonmark.org/ CommonMark
- * @link http://git.io/8WtRvQ JavaScript test runner
- */
-
-use PHPUnit\Framework\TestCase;
-
-class CommonMarkTest extends TestCase
-{
-    const SPEC_URL = 'https://raw.githubusercontent.com/jgm/stmd/master/spec.txt';
-
-    /**
-     * @dataProvider data
-     * @param $section
-     * @param $markdown
-     * @param $expectedHtml
-     */
-    function test_($section, $markdown, $expectedHtml)
-    {
-        $Parsedown = new Parsedown();
-        $Parsedown->setUrlsLinked(false);
-
-        $actualHtml = $Parsedown->text($markdown);
-        $actualHtml = $this->normalizeMarkup($actualHtml);
-
-        $this->assertEquals($expectedHtml, $actualHtml);
-    }
-
-    function data()
-    {
-        $spec = file_get_contents(self::SPEC_URL);
-        $spec = strstr($spec, '<!-- END TESTS -->', true);
-
-        $tests = array();
-        $currentSection = '';
-
-        preg_replace_callback(
-            '/^\.\n([\s\S]*?)^\.\n([\s\S]*?)^\.$|^#{1,6} *(.*)$/m',
-            function($matches) use ( & $tests, & $currentSection, & $testCount) {
-                if (isset($matches[3]) and $matches[3]) {
-                    $currentSection = $matches[3];
-                } else {
-                    $testCount++;
-                    $markdown = $matches[1];
-                    $markdown = preg_replace('/→/', "\t", $markdown);
-                    $expectedHtml = $matches[2];
-                    $expectedHtml = $this->normalizeMarkup($expectedHtml);
-                    $tests []= array(
-                        $currentSection, # section
-                        $markdown, # markdown
-                        $expectedHtml, # html
-                    );
-                }
-            },
-            $spec
-        );
-
-        return $tests;
-    }
-
-    private function normalizeMarkup($markup)
-    {
-        $markup = preg_replace("/\n+/", "\n", $markup);
-        $markup = preg_replace('/^\s+/m', '', $markup);
-        $markup = preg_replace('/^((?:<[\w]+>)+)\n/m', '$1', $markup);
-        $markup = preg_replace('/\n((?:<\/[\w]+>)+)$/m', '$1', $markup);
-        $markup = trim($markup);
-
-        return $markup;
-    }
-}

test/CommonMarkTestStrict.php

@@ -0,0 +1,71 @@
+<?php
+
+/**
+ * Test Parsedown against the CommonMark spec
+ *
+ * @link http://commonmark.org/ CommonMark
+ */
+class CommonMarkTestStrict extends PHPUnit_Framework_TestCase
+{
+    const SPEC_URL = 'https://raw.githubusercontent.com/jgm/CommonMark/master/spec.txt';
+
+    protected $parsedown;
+
+    protected function setUp()
+    {
+        $this->parsedown = new TestParsedown();
+        $this->parsedown->setUrlsLinked(false);
+    }
+
+    /**
+     * @dataProvider data
+     * @param $id
+     * @param $section
+     * @param $markdown
+     * @param $expectedHtml
+     */
+    public function testExample($id, $section, $markdown, $expectedHtml)
+    {
+        $actualHtml = $this->parsedown->text($markdown);
+        $this->assertEquals($expectedHtml, $actualHtml);
+    }
+
+    /**
+     * @return array
+     */
+    public function data()
+    {
+        $spec = file_get_contents(self::SPEC_URL);
+        if ($spec === false) {
+            $this->fail('Unable to load CommonMark spec from ' . self::SPEC_URL);
+        }
+
+        $spec = str_replace("\r\n", "\n", $spec);
+        $spec = strstr($spec, '<!-- END TESTS -->', true);
+
+        $matches = array();
+        preg_match_all('/^`{32} example\n((?s).*?)\n\.\n(?:|((?s).*?)\n)`{32}$|^#{1,6} *(.*?)$/m', $spec, $matches, PREG_SET_ORDER);
+
+        $data = array();
+        $currentId = 0;
+        $currentSection = '';
+        foreach ($matches as $match) {
+            if (isset($match[3])) {
+                $currentSection = $match[3];
+            } else {
+                $currentId++;
+                $markdown = str_replace('→', "\t", $match[1]);
+                $expectedHtml = isset($match[2]) ? str_replace('→', "\t", $match[2]) : '';
+
+                $data[$currentId] = array(
+                    'id' => $currentId,
+                    'section' => $currentSection,
+                    'markdown' => $markdown,
+                    'expectedHtml' => $expectedHtml
+                );
+            }
+        }
+
+        return $data;
+    }
+}

test/CommonMarkTestWeak.php

@@ -0,0 +1,63 @@
+<?php
+require_once(__DIR__ . '/CommonMarkTestStrict.php');
+
+/**
+ * Test Parsedown against the CommonMark spec, but less aggressive
+ *
+ * The resulting HTML markup is cleaned up before comparison, so examples
+ * which would normally fail due to actually invisible differences (e.g.
+ * superfluous whitespaces), don't fail. However, cleanup relies on block
+ * element detection. The detection doesn't work correctly when a element's
+ * `display` CSS property is manipulated. According to that this test is only
+ * a interim solution on Parsedown's way to full CommonMark compatibility.
+ *
+ * @link http://commonmark.org/ CommonMark
+ */
+class CommonMarkTestWeak extends CommonMarkTestStrict
+{
+    protected $textLevelElementRegex;
+
+    protected function setUp()
+    {
+        parent::setUp();
+
+        $textLevelElements = $this->parsedown->getTextLevelElements();
+        array_walk($textLevelElements, function (&$element) {
+            $element = preg_quote($element, '/');
+        });
+        $this->textLevelElementRegex = '\b(?:' . implode('|', $textLevelElements) . ')\b';
+    }
+
+    /**
+     * @dataProvider data
+     * @param $id
+     * @param $section
+     * @param $markdown
+     * @param $expectedHtml
+     */
+    public function testExample($id, $section, $markdown, $expectedHtml)
+    {
+        $expectedHtml = $this->cleanupHtml($expectedHtml);
+
+        $actualHtml = $this->parsedown->text($markdown);
+        $actualHtml = $this->cleanupHtml($actualHtml);
+
+        $this->assertEquals($expectedHtml, $actualHtml);
+    }
+
+    protected function cleanupHtml($markup)
+    {
+        // invisible whitespaces at the beginning and end of block elements
+        // however, whitespaces at the beginning of <pre> elements do matter
+        $markup = preg_replace(
+            array(
+                '/(<(?!(?:' . $this->textLevelElementRegex . '|\bpre\b))\w+\b[^>]*>(?:<' . $this->textLevelElementRegex . '[^>]*>)*)\s+/s',
+                '/\s+((?:<\/' . $this->textLevelElementRegex . '>)*<\/(?!' . $this->textLevelElementRegex . ')\w+\b>)/s'
+            ),
+            '$1',
+            $markup
+        );
+
+        return $markup;
+    }
+}

test/ParsedownTest.php

@@ -29,7 +29,7 @@ class ParsedownTest extends TestCase
      */
     protected function initParsedown()
     {
-        $Parsedown = new Parsedown();
+        $Parsedown = new TestParsedown();
 
         return $Parsedown;
     }
@@ -136,15 +136,14 @@ color: red;
 <p>comment</p>
 <p>&lt;!-- html comment --&gt;</p>
 EXPECTED_HTML;
-        $parsedownWithNoMarkup = new Parsedown();
+
+        $parsedownWithNoMarkup = new TestParsedown();
         $parsedownWithNoMarkup->setMarkupEscaped(true);
         $this->assertEquals($expectedHtml, $parsedownWithNoMarkup->text($markdownWithHtml));
     }
 
     public function testLateStaticBinding()
     {
-        include __DIR__ . '/TestParsedown.php';
-
         $parsedown = Parsedown::instance();
         $this->assertInstanceOf('Parsedown', $parsedown);
 

test/TestParsedown.php

@@ -2,4 +2,8 @@
 
 class TestParsedown extends Parsedown
 {
+    public function getTextLevelElements()
+    {
+        return $this->textLevelElements;
+    }
 }

test/bootstrap.php

@@ -1,3 +0,0 @@
-<?php
-
-include 'Parsedown.php';

test/data/fenced_code_block.html

@@ -3,4 +3,9 @@
 $message = 'fenced code block';
 echo $message;</code></pre>
 <pre><code>tilde</code></pre>
-<pre><code class="language-php">echo 'language identifier';</code></pre>
+<pre><code class="language-php">echo 'language identifier';</code></pre>
+<pre><code class="language-c#">echo 'language identifier with non words';</code></pre>
+<pre><code class="language-html+php">&lt;?php
+echo "Hello World";
+?&gt;
+&lt;a href="http://auraphp.com" &gt;Aura Project&lt;/a&gt;</code></pre>

test/data/fenced_code_block.md

@@ -11,4 +11,15 @@ tilde
 
 ```php
 echo 'language identifier';
+```
+
+```c#
+echo 'language identifier with non words';
+```
+
+```html+php
+<?php
+echo "Hello World";
+?>
+<a href="http://auraphp.com" >Aura Project</a>
 ```

test/data/multiline_lists.html

@@ -0,0 +1,10 @@
+<ol>
+<li>
+<p>One
+First body copy</p>
+</li>
+<li>
+<p>Two
+Last body copy</p>
+</li>
+</ol>

test/data/multiline_lists.md

@@ -0,0 +1,5 @@
+1. One
+   First body copy
+
+2. Two
+   Last body copy

test/data/paragraph_list.html

@@ -8,5 +8,7 @@
 <li>
 <p>li</p>
 </li>
-<li>li</li>
+<li>
+<p>li</p>
+</li>
 </ul>

test/data/sparse_dense_list.html

@@ -2,6 +2,10 @@
 <li>
 <p>li</p>
 </li>
-<li>li</li>
+<li>
-<li>li</li>
+<p>li</p>
+</li>
+<li>
+<p>li</p>
+</li>
 </ul>

test/data/sparse_list.html

@@ -2,7 +2,9 @@
 <li>
 <p>li</p>
 </li>
-<li>li</li>
+<li>
+<p>li</p>
+</li>
 </ul>
 <hr />
 <ul>