Prepare for Goldmark

This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo.

This introduces a new `markup` package with some common interfaces and each implementation in its own package.

See #5963

markup/asciidoc/convert.go

@@ -0,0 +1,97 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package asciidoc converts Asciidoc to HTML using Asciidoc or Asciidoctor
+// external binaries.
+package asciidoc
+
+import (
+	"os/exec"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &asciidocConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+}
+
+type asciidocConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil
+}
+
+// getAsciidocContent calls asciidoctor or asciidoc as an external helper
+// to convert AsciiDoc content to HTML.
+func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte {
+	var isAsciidoctor bool
+	path := getAsciidoctorExecPath()
+	if path == "" {
+		path = getAsciidocExecPath()
+		if path == "" {
+			a.cfg.Logger.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
+				"                 Leaving AsciiDoc content unrendered.")
+			return src
+		}
+	} else {
+		isAsciidoctor = true
+	}
+
+	a.cfg.Logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+	args := []string{"--no-header-footer", "--safe"}
+	if isAsciidoctor {
+		// asciidoctor-specific arg to show stack traces on errors
+		args = append(args, "--trace")
+	}
+	args = append(args, "-")
+	return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args)
+}
+
+func getAsciidocExecPath() string {
+	path, err := exec.LookPath("asciidoc")
+	if err != nil {
+		return ""
+	}
+	return path
+}
+
+func getAsciidoctorExecPath() string {
+	path, err := exec.LookPath("asciidoctor")
+	if err != nil {
+		return ""
+	}
+	return path
+}
+
+// Supports returns whether Asciidoc or Asciidoctor is installed on this computer.
+func Supports() bool {
+	return (getAsciidoctorExecPath() != "" ||
+		getAsciidocExecPath() != "")
+}

markup/asciidoc/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package asciidoc
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("asciidoc/asciidoctor not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"paragraph\">\n<p>testContent</p>\n</div>\n")
+}

markup/blackfriday/convert.go

@@ -0,0 +1,224 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package blackfriday converts Markdown to HTML using Blackfriday v1.
+package blackfriday
+
+import (
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/gohugoio/hugo/markup/internal"
+	"github.com/russross/blackfriday"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
+
+	pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
+	pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
+	pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
+
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		b := defaultBlackFriday
+		extensions := defaultExtensions
+
+		if ctx.ConfigOverrides != nil {
+			var err error
+			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			if err != nil {
+				return nil, err
+			}
+			extensions = getMarkdownExtensions(b)
+		}
+
+		return &blackfridayConverter{
+			ctx:        ctx,
+			bf:         b,
+			extensions: extensions,
+			cfg:        cfg,
+
+			pygmentsCodeFences:            pygmentsCodeFences,
+			pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
+			pygmentsOptions:               pygmentsOptions,
+		}, nil
+	}
+
+	return n, nil
+
+}
+
+type blackfridayConverter struct {
+	ctx        converter.DocumentContext
+	bf         *internal.BlackFriday
+	extensions int
+
+	pygmentsCodeFences            bool
+	pygmentsCodeFencesGuessSyntax bool
+	pygmentsOptions               string
+
+	cfg converter.ProviderConfig
+}
+
+func (c *blackfridayConverter) AnchorSuffix() string {
+	if c.bf.PlainIDAnchors {
+		return ""
+	}
+	return ":" + c.ctx.DocumentID
+}
+
+func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	r := c.getHTMLRenderer(ctx.RenderTOC)
+
+	return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
+
+}
+
+func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
+	flags := getFlags(renderTOC, c.bf)
+
+	documentID := c.ctx.DocumentID
+
+	renderParameters := blackfriday.HtmlRendererParameters{
+		FootnoteAnchorPrefix:       c.bf.FootnoteAnchorPrefix,
+		FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents,
+	}
+
+	if documentID != "" && !c.bf.PlainIDAnchors {
+		renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+		renderParameters.HeaderIDSuffix = ":" + documentID
+	}
+
+	return &hugoHTMLRenderer{
+		c:        c,
+		Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters),
+	}
+}
+
+func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
+
+	var flags int
+
+	if renderTOC {
+		flags = blackfriday.HTML_TOC
+	}
+
+	flags |= blackfriday.HTML_USE_XHTML
+	flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
+
+	if cfg.Smartypants {
+		flags |= blackfriday.HTML_USE_SMARTYPANTS
+	}
+
+	if cfg.SmartypantsQuotesNBSP {
+		flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
+	}
+
+	if cfg.AngledQuotes {
+		flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
+	}
+
+	if cfg.Fractions {
+		flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
+	}
+
+	if cfg.HrefTargetBlank {
+		flags |= blackfriday.HTML_HREF_TARGET_BLANK
+	}
+
+	if cfg.NofollowLinks {
+		flags |= blackfriday.HTML_NOFOLLOW_LINKS
+	}
+
+	if cfg.NoreferrerLinks {
+		flags |= blackfriday.HTML_NOREFERRER_LINKS
+	}
+
+	if cfg.SmartDashes {
+		flags |= blackfriday.HTML_SMARTYPANTS_DASHES
+	}
+
+	if cfg.LatexDashes {
+		flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
+	}
+
+	if cfg.SkipHTML {
+		flags |= blackfriday.HTML_SKIP_HTML
+	}
+
+	return flags
+}
+
+func getMarkdownExtensions(cfg *internal.BlackFriday) int {
+	// Default Blackfriday common extensions
+	commonExtensions := 0 |
+		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
+		blackfriday.EXTENSION_TABLES |
+		blackfriday.EXTENSION_FENCED_CODE |
+		blackfriday.EXTENSION_AUTOLINK |
+		blackfriday.EXTENSION_STRIKETHROUGH |
+		blackfriday.EXTENSION_SPACE_HEADERS |
+		blackfriday.EXTENSION_HEADER_IDS |
+		blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
+		blackfriday.EXTENSION_DEFINITION_LISTS
+
+	// Extra Blackfriday extensions that Hugo enables by default
+	flags := commonExtensions |
+		blackfriday.EXTENSION_AUTO_HEADER_IDS |
+		blackfriday.EXTENSION_FOOTNOTES
+
+	for _, extension := range cfg.Extensions {
+		if flag, ok := blackfridayExtensionMap[extension]; ok {
+			flags |= flag
+		}
+	}
+	for _, extension := range cfg.ExtensionsMask {
+		if flag, ok := blackfridayExtensionMap[extension]; ok {
+			flags &= ^flag
+		}
+	}
+	return flags
+}
+
+var blackfridayExtensionMap = map[string]int{
+	"noIntraEmphasis":        blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
+	"tables":                 blackfriday.EXTENSION_TABLES,
+	"fencedCode":             blackfriday.EXTENSION_FENCED_CODE,
+	"autolink":               blackfriday.EXTENSION_AUTOLINK,
+	"strikethrough":          blackfriday.EXTENSION_STRIKETHROUGH,
+	"laxHtmlBlocks":          blackfriday.EXTENSION_LAX_HTML_BLOCKS,
+	"spaceHeaders":           blackfriday.EXTENSION_SPACE_HEADERS,
+	"hardLineBreak":          blackfriday.EXTENSION_HARD_LINE_BREAK,
+	"tabSizeEight":           blackfriday.EXTENSION_TAB_SIZE_EIGHT,
+	"footnotes":              blackfriday.EXTENSION_FOOTNOTES,
+	"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+	"headerIds":              blackfriday.EXTENSION_HEADER_IDS,
+	"titleblock":             blackfriday.EXTENSION_TITLEBLOCK,
+	"autoHeaderIds":          blackfriday.EXTENSION_AUTO_HEADER_IDS,
+	"backslashLineBreak":     blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
+	"definitionLists":        blackfriday.EXTENSION_DEFINITION_LISTS,
+	"joinLines":              blackfriday.EXTENSION_JOIN_LINES,
+}
+
+var (
+	_ converter.DocumentInfo = (*blackfridayConverter)(nil)
+)

markup/blackfriday/convert_test.go

@@ -0,0 +1,194 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blackfriday
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/russross/blackfriday"
+)
+
+func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{"headerId"}
+	b.ExtensionsMask = []string{"noIntraEmphasis"}
+
+	actualFlags := getMarkdownExtensions(b)
+	if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
+		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
+	}
+}
+
+func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
+	type data struct {
+		testFlag int
+	}
+
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{""}
+	b.ExtensionsMask = []string{""}
+	allExtensions := []data{
+		{blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
+		{blackfriday.EXTENSION_TABLES},
+		{blackfriday.EXTENSION_FENCED_CODE},
+		{blackfriday.EXTENSION_AUTOLINK},
+		{blackfriday.EXTENSION_STRIKETHROUGH},
+		// {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
+		{blackfriday.EXTENSION_SPACE_HEADERS},
+		// {blackfriday.EXTENSION_HARD_LINE_BREAK},
+		// {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
+		{blackfriday.EXTENSION_FOOTNOTES},
+		// {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+		{blackfriday.EXTENSION_HEADER_IDS},
+		// {blackfriday.EXTENSION_TITLEBLOCK},
+		{blackfriday.EXTENSION_AUTO_HEADER_IDS},
+		{blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
+		{blackfriday.EXTENSION_DEFINITION_LISTS},
+	}
+
+	actualFlags := getMarkdownExtensions(b)
+	for _, e := range allExtensions {
+		if actualFlags&e.testFlag != e.testFlag {
+			t.Errorf("Flag %v was not found in the list of extensions.", e)
+		}
+	}
+}
+
+func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{"definitionLists"}
+	b.ExtensionsMask = []string{""}
+
+	actualFlags := getMarkdownExtensions(b)
+	if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
+		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
+	}
+}
+
+func TestGetFlags(t *testing.T) {
+	c := qt.New(t)
+	cfg := converter.ProviderConfig{Cfg: viper.New()}
+	b, err := internal.NewBlackfriday(cfg)
+	c.Assert(err, qt.IsNil)
+	flags := getFlags(false, b)
+	if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
+		t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
+	}
+}
+
+func TestGetAllFlags(t *testing.T) {
+	c := qt.New(t)
+	cfg := converter.ProviderConfig{Cfg: viper.New()}
+	b, err := internal.NewBlackfriday(cfg)
+	c.Assert(err, qt.IsNil)
+
+	type data struct {
+		testFlag int
+	}
+
+	allFlags := []data{
+		{blackfriday.HTML_USE_XHTML},
+		{blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
+		{blackfriday.HTML_USE_SMARTYPANTS},
+		{blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
+		{blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
+		{blackfriday.HTML_SMARTYPANTS_FRACTIONS},
+		{blackfriday.HTML_HREF_TARGET_BLANK},
+		{blackfriday.HTML_NOFOLLOW_LINKS},
+		{blackfriday.HTML_NOREFERRER_LINKS},
+		{blackfriday.HTML_SMARTYPANTS_DASHES},
+		{blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
+	}
+
+	b.AngledQuotes = true
+	b.Fractions = true
+	b.HrefTargetBlank = true
+	b.NofollowLinks = true
+	b.NoreferrerLinks = true
+	b.LatexDashes = true
+	b.PlainIDAnchors = true
+	b.SmartDashes = true
+	b.Smartypants = true
+	b.SmartypantsQuotesNBSP = true
+
+	actualFlags := getFlags(false, b)
+
+	var expectedFlags int
+	//OR-ing flags together...
+	for _, d := range allFlags {
+		expectedFlags |= d.testFlag
+	}
+	if expectedFlags != actualFlags {
+		t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
+	}
+}
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{
+		Cfg: viper.New(),
+	})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
+
+func TestGetHTMLRendererAnchors(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{
+		Cfg: viper.New(),
+	})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{
+		DocumentID: "testid",
+		ConfigOverrides: map[string]interface{}{
+			"plainIDAnchors": false,
+			"footnotes":      true,
+		},
+	})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header
+
+This is a footnote.[^1] And then some.
+
+
+[^1]: Footnote text.
+
+`)})
+
+	c.Assert(err, qt.IsNil)
+	s := string(b.Bytes())
+	c.Assert(s, qt.Contains, "<h1 id=\"header:testid\">Header</h1>")
+	c.Assert(s, qt.Contains, "This is a footnote.<sup class=\"footnote-ref\" id=\"fnref:testid:1\"><a href=\"#fn:testid:1\">1</a></sup>")
+	c.Assert(s, qt.Contains, "<a class=\"footnote-return\" href=\"#fnref:testid:1\"><sup>[return]</sup></a>")
+}

markup/blackfriday/renderer.go

@@ -0,0 +1,85 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blackfriday
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/russross/blackfriday"
+)
+
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// adding some custom behaviour.
+type hugoHTMLRenderer struct {
+	c *blackfridayConverter
+	blackfriday.Renderer
+}
+
+// BlockCode renders a given text as a block of code.
+// Pygments is used if it is setup to handle code fences.
+func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
+	if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
+		opts := r.c.pygmentsOptions
+		str := strings.Trim(string(text), "\n\r")
+		highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
+		out.WriteString(highlighted)
+	} else {
+		r.Renderer.BlockCode(out, text, lang)
+	}
+}
+
+// ListItem adds task list support to the Blackfriday renderer.
+func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
+	if !r.c.bf.TaskLists {
+		r.Renderer.ListItem(out, text, flags)
+		return
+	}
+
+	switch {
+	case bytes.HasPrefix(text, []byte("[ ] ")):
+		text = append([]byte(`<label><input type="checkbox" disabled class="task-list-item">`), text[3:]...)
+		text = append(text, []byte(`</label>`)...)
+
+	case bytes.HasPrefix(text, []byte("[x] ")) || bytes.HasPrefix(text, []byte("[X] ")):
+		text = append([]byte(`<label><input type="checkbox" checked disabled class="task-list-item">`), text[3:]...)
+		text = append(text, []byte(`</label>`)...)
+	}
+
+	r.Renderer.ListItem(out, text, flags)
+}
+
+// List adds task list support to the Blackfriday renderer.
+func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
+	if !r.c.bf.TaskLists {
+		r.Renderer.List(out, text, flags)
+		return
+	}
+	marker := out.Len()
+	r.Renderer.List(out, text, flags)
+	if out.Len() > marker {
+		list := out.Bytes()[marker:]
+		if bytes.Contains(list, []byte("task-list-item")) {
+			// Find the index of the first >, it might be 3 or 4 depending on whether
+			// there is a new line at the start, but this is safer than just hardcoding it.
+			closingBracketIndex := bytes.Index(list, []byte(">"))
+			// Rewrite the buffer from the marker
+			out.Truncate(marker)
+			// Safely assuming closingBracketIndex won't be -1 since there is a list
+			// May be either dl, ul or ol
+			list := append(list[:closingBracketIndex], append([]byte(` class="task-list"`), list[closingBracketIndex:]...)...)
+			out.Write(list)
+		}
+	}
+}

markup/converter/converter.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter
+
+import (
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+// ProviderConfig configures a new Provider.
+type ProviderConfig struct {
+	Cfg       config.Provider // Site config
+	ContentFs afero.Fs
+	Logger    *loggers.Logger
+	Highlight func(code, lang, optsStr string) (string, error)
+}
+
+// NewProvider creates converter providers.
+type NewProvider interface {
+	New(cfg ProviderConfig) (Provider, error)
+}
+
+// Provider creates converters.
+type Provider interface {
+	New(ctx DocumentContext) (Converter, error)
+}
+
+// NewConverter is an adapter that can be used as a ConverterProvider.
+type NewConverter func(ctx DocumentContext) (Converter, error)
+
+// New creates a new Converter for the given ctx.
+func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
+	return n(ctx)
+}
+
+// Converter wraps the Convert method that converts some markup into
+// another format, e.g. Markdown to HTML.
+type Converter interface {
+	Convert(ctx RenderContext) (Result, error)
+}
+
+// Result represents the minimum returned from Convert.
+type Result interface {
+	Bytes() []byte
+}
+
+// DocumentInfo holds additional information provided by some converters.
+type DocumentInfo interface {
+	AnchorSuffix() string
+}
+
+// Bytes holds a byte slice and implements the Result interface.
+type Bytes []byte
+
+// Bytes returns itself
+func (b Bytes) Bytes() []byte {
+	return b
+}
+
+// DocumentContext holds contextual information about the document to convert.
+type DocumentContext struct {
+	DocumentID      string
+	DocumentName    string
+	ConfigOverrides map[string]interface{}
+}
+
+// RenderContext holds contextual information about the content to render.
+type RenderContext struct {
+	Src       []byte
+	RenderTOC bool
+}

markup/internal/blackfriday.go

@@ -0,0 +1,108 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package helpers implements general utility functions that work with
+// and on content.  The helper functions defined here lay down the
+// foundation of how Hugo works with files and filepaths, and perform
+// string operations on content.
+
+package internal
+
+import (
+	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+// BlackFriday holds configuration values for BlackFriday rendering.
+// It is kept here because it's used in several packages.
+type BlackFriday struct {
+	Smartypants           bool
+	SmartypantsQuotesNBSP bool
+	AngledQuotes          bool
+	Fractions             bool
+	HrefTargetBlank       bool
+	NofollowLinks         bool
+	NoreferrerLinks       bool
+	SmartDashes           bool
+	LatexDashes           bool
+	TaskLists             bool
+	PlainIDAnchors        bool
+	Extensions            []string
+	ExtensionsMask        []string
+	SkipHTML              bool
+
+	FootnoteAnchorPrefix       string
+	FootnoteReturnLinkContents string
+}
+
+func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
+	// Create a copy so we can modify it.
+	bf := *old
+	if err := mapstructure.Decode(m, &bf); err != nil {
+		return nil, errors.WithMessage(err, "failed to decode rendering config")
+	}
+	return &bf, nil
+}
+
+// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
+func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
+	var siteConfig map[string]interface{}
+	if cfg.Cfg != nil {
+		siteConfig = cfg.Cfg.GetStringMap("blackfriday")
+	}
+
+	defaultParam := map[string]interface{}{
+		"smartypants":           true,
+		"angledQuotes":          false,
+		"smartypantsQuotesNBSP": false,
+		"fractions":             true,
+		"hrefTargetBlank":       false,
+		"nofollowLinks":         false,
+		"noreferrerLinks":       false,
+		"smartDashes":           true,
+		"latexDashes":           true,
+		"plainIDAnchors":        true,
+		"taskLists":             true,
+		"skipHTML":              false,
+	}
+
+	maps.ToLower(defaultParam)
+
+	config := make(map[string]interface{})
+
+	for k, v := range defaultParam {
+		config[k] = v
+	}
+
+	for k, v := range siteConfig {
+		config[k] = v
+	}
+
+	combinedConfig := &BlackFriday{}
+	if err := mapstructure.Decode(config, combinedConfig); err != nil {
+		return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
+	}
+
+	// TODO(bep) update/consolidate docs
+	if combinedConfig.FootnoteAnchorPrefix == "" {
+		combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
+	}
+
+	if combinedConfig.FootnoteReturnLinkContents == "" {
+		combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
+	}
+
+	return combinedConfig, nil
+}

markup/internal/external.go

@@ -0,0 +1,52 @@
+package internal
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+func ExternallyRenderContent(
+	cfg converter.ProviderConfig,
+	ctx converter.DocumentContext,
+	content []byte, path string, args []string) []byte {
+
+	logger := cfg.Logger
+	cmd := exec.Command(path, args...)
+	cmd.Stdin = bytes.NewReader(content)
+	var out, cmderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &cmderr
+	err := cmd.Run()
+	// Most external helpers exit w/ non-zero exit code only if severe, i.e.
+	// halting errors occurred. -> log stderr output regardless of state of err
+	for _, item := range strings.Split(cmderr.String(), "\n") {
+		item := strings.TrimSpace(item)
+		if item != "" {
+			logger.ERROR.Printf("%s: %s", ctx.DocumentName, item)
+		}
+	}
+	if err != nil {
+		logger.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
+	}
+
+	return normalizeExternalHelperLineFeeds(out.Bytes())
+}
+
+// Strips carriage returns from third-party / external processes (useful for Windows)
+func normalizeExternalHelperLineFeeds(content []byte) []byte {
+	return bytes.Replace(content, []byte("\r"), []byte(""), -1)
+}
+
+func GetPythonExecPath() string {
+	path, err := exec.LookPath("python")
+	if err != nil {
+		path, err = exec.LookPath("python.exe")
+		if err != nil {
+			return ""
+		}
+	}
+	return path
+}

markup/markup.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+	"strings"
+
+	"github.com/gohugoio/hugo/markup/org"
+
+	"github.com/gohugoio/hugo/markup/asciidoc"
+	"github.com/gohugoio/hugo/markup/blackfriday"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/gohugoio/hugo/markup/mmark"
+	"github.com/gohugoio/hugo/markup/pandoc"
+	"github.com/gohugoio/hugo/markup/rst"
+)
+
+func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
+	converters := make(map[string]converter.Provider)
+
+	add := func(p converter.NewProvider, aliases ...string) error {
+		c, err := p.New(cfg)
+		if err != nil {
+			return err
+		}
+		addConverter(converters, c, aliases...)
+		return nil
+	}
+
+	if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
+		return nil, err
+	}
+	if err := add(mmark.Provider, "mmark"); err != nil {
+		return nil, err
+	}
+	if err := add(asciidoc.Provider, "asciidoc"); err != nil {
+		return nil, err
+	}
+	if err := add(rst.Provider, "rst"); err != nil {
+		return nil, err
+	}
+	if err := add(pandoc.Provider, "pandoc"); err != nil {
+		return nil, err
+	}
+	if err := add(org.Provider, "org"); err != nil {
+		return nil, err
+	}
+
+	return &converterRegistry{converters: converters}, nil
+}
+
+type ConverterProvider interface {
+	Get(name string) converter.Provider
+}
+
+type converterRegistry struct {
+	// Maps name (md, markdown, blackfriday etc.) to a converter provider.
+	// Note that this is also used for aliasing, so the same converter
+	// may be registered multiple times.
+	// All names are lower case.
+	converters map[string]converter.Provider
+}
+
+func (r *converterRegistry) Get(name string) converter.Provider {
+	return r.converters[strings.ToLower(name)]
+}
+
+func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
+	for _, alias := range aliases {
+		m[alias] = c
+	}
+}

markup/markup_test.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConverterRegistry(t *testing.T) {
+	c := qt.New(t)
+
+	r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
+
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(r.Get("foo"), qt.IsNil)
+	c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("org"), qt.Not(qt.IsNil))
+
+}

markup/mmark/convert.go

@@ -0,0 +1,143 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mmark converts Markdown to HTML using MMark v1.
+package mmark
+
+import (
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/miekg/mmark"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	defaultExtensions := getMmarkExtensions(defaultBlackFriday)
+
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		b := defaultBlackFriday
+		extensions := defaultExtensions
+
+		if ctx.ConfigOverrides != nil {
+			var err error
+			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			if err != nil {
+				return nil, err
+			}
+			extensions = getMmarkExtensions(b)
+		}
+
+		return &mmarkConverter{
+			ctx:        ctx,
+			b:          b,
+			extensions: extensions,
+			cfg:        cfg,
+		}, nil
+	}
+
+	return n, nil
+
+}
+
+type mmarkConverter struct {
+	ctx        converter.DocumentContext
+	extensions int
+	b          *internal.BlackFriday
+	cfg        converter.ProviderConfig
+}
+
+func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	r := getHTMLRenderer(c.ctx, c.b, c.cfg)
+	return mmark.Parse(ctx.Src, r, c.extensions), nil
+}
+
+func getHTMLRenderer(
+	ctx converter.DocumentContext,
+	cfg *internal.BlackFriday,
+	pcfg converter.ProviderConfig) mmark.Renderer {
+
+	var (
+		flags      int
+		documentID string
+	)
+
+	documentID = ctx.DocumentID
+
+	renderParameters := mmark.HtmlRendererParameters{
+		FootnoteAnchorPrefix:       cfg.FootnoteAnchorPrefix,
+		FootnoteReturnLinkContents: cfg.FootnoteReturnLinkContents,
+	}
+
+	if documentID != "" && !cfg.PlainIDAnchors {
+		renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+	}
+
+	htmlFlags := flags
+	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
+
+	return &mmarkRenderer{
+		Config:    cfg,
+		Cfg:       pcfg.Cfg,
+		highlight: pcfg.Highlight,
+		Renderer:  mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
+	}
+
+}
+
+func getMmarkExtensions(cfg *internal.BlackFriday) int {
+	flags := 0
+	flags |= mmark.EXTENSION_TABLES
+	flags |= mmark.EXTENSION_FENCED_CODE
+	flags |= mmark.EXTENSION_AUTOLINK
+	flags |= mmark.EXTENSION_SPACE_HEADERS
+	flags |= mmark.EXTENSION_CITATION
+	flags |= mmark.EXTENSION_TITLEBLOCK_TOML
+	flags |= mmark.EXTENSION_HEADER_IDS
+	flags |= mmark.EXTENSION_AUTO_HEADER_IDS
+	flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
+	flags |= mmark.EXTENSION_FOOTNOTES
+	flags |= mmark.EXTENSION_SHORT_REF
+	flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
+	flags |= mmark.EXTENSION_INCLUDE
+
+	for _, extension := range cfg.Extensions {
+		if flag, ok := mmarkExtensionMap[extension]; ok {
+			flags |= flag
+		}
+	}
+	return flags
+}
+
+var mmarkExtensionMap = map[string]int{
+	"tables":                 mmark.EXTENSION_TABLES,
+	"fencedCode":             mmark.EXTENSION_FENCED_CODE,
+	"autolink":               mmark.EXTENSION_AUTOLINK,
+	"laxHtmlBlocks":          mmark.EXTENSION_LAX_HTML_BLOCKS,
+	"spaceHeaders":           mmark.EXTENSION_SPACE_HEADERS,
+	"hardLineBreak":          mmark.EXTENSION_HARD_LINE_BREAK,
+	"footnotes":              mmark.EXTENSION_FOOTNOTES,
+	"noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+	"headerIds":              mmark.EXTENSION_HEADER_IDS,
+	"autoHeaderIds":          mmark.EXTENSION_AUTO_HEADER_IDS,
+}

markup/mmark/convert_test.go

@@ -0,0 +1,77 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/miekg/mmark"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestGetMmarkExtensions(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	//TODO: This is doing the same just with different marks...
+	type data struct {
+		testFlag int
+	}
+
+	b.Extensions = []string{"tables"}
+	b.ExtensionsMask = []string{""}
+	allExtensions := []data{
+		{mmark.EXTENSION_TABLES},
+		{mmark.EXTENSION_FENCED_CODE},
+		{mmark.EXTENSION_AUTOLINK},
+		{mmark.EXTENSION_SPACE_HEADERS},
+		{mmark.EXTENSION_CITATION},
+		{mmark.EXTENSION_TITLEBLOCK_TOML},
+		{mmark.EXTENSION_HEADER_IDS},
+		{mmark.EXTENSION_AUTO_HEADER_IDS},
+		{mmark.EXTENSION_UNIQUE_HEADER_IDS},
+		{mmark.EXTENSION_FOOTNOTES},
+		{mmark.EXTENSION_SHORT_REF},
+		{mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+		{mmark.EXTENSION_INCLUDE},
+	}
+
+	actualFlags := getMmarkExtensions(b)
+	for _, e := range allExtensions {
+		if actualFlags&e.testFlag != e.testFlag {
+			t.Errorf("Flag %v was not found in the list of extensions.", e)
+		}
+	}
+}
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Cfg: viper.New(), Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}

markup/mmark/renderer.go

@@ -0,0 +1,44 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/miekg/mmark"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/markup/internal"
+)
+
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// adding some custom behaviour.
+type mmarkRenderer struct {
+	Cfg       config.Provider
+	Config    *internal.BlackFriday
+	highlight func(code, lang, optsStr string) (string, error)
+	mmark.Renderer
+}
+
+// BlockCode renders a given text as a block of code.
+func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
+	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
+		str := strings.Trim(string(text), "\n\r")
+		highlighted, _ := r.highlight(str, lang, "")
+		out.WriteString(highlighted)
+	} else {
+		r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
+	}
+}

markup/org/convert.go

@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package org converts Emacs Org-Mode to HTML.
+package org
+
+import (
+	"bytes"
+
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/niklasfasching/go-org/org"
+	"github.com/spf13/afero"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provide{}
+
+type provide struct {
+}
+
+func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &orgConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+}
+
+type orgConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	logger := c.cfg.Logger
+	config := org.New()
+	config.Log = logger.WARN
+	config.ReadFile = func(filename string) ([]byte, error) {
+		return afero.ReadFile(c.cfg.ContentFs, filename)
+	}
+	writer := org.NewHTMLWriter()
+	writer.HighlightCodeBlock = func(source, lang string) string {
+		highlightedSource, err := c.cfg.Highlight(source, lang, "")
+		if err != nil {
+			logger.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
+			return source
+		}
+		return highlightedSource
+	}
+
+	html, err := config.Parse(bytes.NewReader(ctx.Src), c.ctx.DocumentName).Write(writer)
+	if err != nil {
+		logger.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
+		return converter.Bytes(ctx.Src), nil
+	}
+	return converter.Bytes([]byte(html)), nil
+}

markup/org/convert_test.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>\ntestContent\n</p>\n")
+}

markup/pandoc/convert.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pandoc converts content to HTML using Pandoc as an external helper.
+package pandoc
+
+import (
+	"os/exec"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &pandocConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+
+}
+
+type pandocConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil
+}
+
+// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
+func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte {
+	logger := c.cfg.Logger
+	path := getPandocExecPath()
+	if path == "" {
+		logger.ERROR.Println("pandoc not found in $PATH: Please install.\n",
+			"                 Leaving pandoc content unrendered.")
+		return src
+	}
+	args := []string{"--mathjax"}
+	return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+}
+
+func getPandocExecPath() string {
+	path, err := exec.LookPath("pandoc")
+	if err != nil {
+		return ""
+	}
+
+	return path
+}
+
+// Supports returns whether Pandoc is installed on this computer.
+func Supports() bool {
+	return getPandocExecPath() != ""
+}

markup/pandoc/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pandoc
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("pandoc not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}

markup/rst/convert.go

@@ -0,0 +1,109 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rst converts content to HTML using the RST external helper.
+package rst
+
+import (
+	"bytes"
+	"os/exec"
+	"runtime"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &rstConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+
+}
+
+type rstConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil
+}
+
+// getRstContent calls the Python script rst2html as an external helper
+// to convert reStructuredText content to HTML.
+func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte {
+	logger := c.cfg.Logger
+	path := getRstExecPath()
+
+	if path == "" {
+		logger.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
+			"                 Leaving reStructuredText content unrendered.")
+		return src
+	}
+	logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+	var result []byte
+	// certain *nix based OSs wrap executables in scripted launchers
+	// invoking binaries on these OSs via python interpreter causes SyntaxError
+	// invoke directly so that shebangs work as expected
+	// handle Windows manually because it doesn't do shebangs
+	if runtime.GOOS == "windows" {
+		python := internal.GetPythonExecPath()
+		args := []string{path, "--leave-comments", "--initial-header-level=2"}
+		result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args)
+	} else {
+		args := []string{"--leave-comments", "--initial-header-level=2"}
+		result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+	}
+	// TODO(bep) check if rst2html has a body only option.
+	bodyStart := bytes.Index(result, []byte("<body>\n"))
+	if bodyStart < 0 {
+		bodyStart = -7 //compensate for length
+	}
+
+	bodyEnd := bytes.Index(result, []byte("\n</body>"))
+	if bodyEnd < 0 || bodyEnd >= len(result) {
+		bodyEnd = len(result) - 1
+		if bodyEnd < 0 {
+			bodyEnd = 0
+		}
+	}
+
+	return result[bodyStart+7 : bodyEnd]
+}
+
+func getRstExecPath() string {
+	path, err := exec.LookPath("rst2html")
+	if err != nil {
+		path, err = exec.LookPath("rst2html.py")
+		if err != nil {
+			return ""
+		}
+	}
+	return path
+}
+
+// Supports returns whether rst is installed on this computer.
+func Supports() bool {
+	return getRstExecPath() != ""
+}

markup/rst/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rst
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("rst not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"document\">\n\n\n<p>testContent</p>\n</div>")
+}