diff --git a/hugolib/page.go b/hugolib/page.go index 578aed9ed..47083e9ef 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -730,56 +730,48 @@ func splitUserDefinedSummaryAndContent(markup string, c []byte) (sc *summaryCont } }() - c = bytes.TrimSpace(c) - startDivider := bytes.Index(c, internalSummaryDivider) + startDivider := bytes.Index(c, internalSummaryDividerBaseBytes) if startDivider == -1 { return } - endDivider := startDivider + len(internalSummaryDivider) - endSummary := startDivider + startTag := "p" + switch markup { + case "asciidoc": + startTag = "div" - var ( - startMarkup []byte - endMarkup []byte - addDiv bool - ) + } + + // Walk back and forward to the surrounding tags. + start := bytes.LastIndex(c[:startDivider], []byte("<"+startTag)) + end := bytes.Index(c[startDivider:], []byte(""+startTag)) + + if start == -1 { + start = startDivider + } else { + start = startDivider - (startDivider - start) + } + + if end == -1 { + end = startDivider + len(internalSummaryDividerBase) + } else { + end = startDivider + end + len(startTag) + 3 + } + + var addDiv bool switch markup { - default: - startMarkup = []byte("
") - endMarkup = []byte("
") - case "asciidoc": - startMarkup = []byte("") - endMarkup = []byte("
") addDiv = true } - // Find the closest end/start markup string to the divider - fromStart := -1 - fromIdx := bytes.LastIndex(c[:startDivider], startMarkup) - if fromIdx != -1 { - fromStart = startDivider - fromIdx - len(startMarkup) - } - fromEnd := bytes.Index(c[endDivider:], endMarkup) + withoutDivider := append(c[:start], bytes.Trim(c[end:], "\n")...) - if fromEnd != -1 && fromEnd <= fromStart { - endSummary = startDivider + fromEnd + len(endMarkup) - } else if fromStart != -1 && fromEnd != -1 { - endSummary = startDivider - fromStart - len(startMarkup) - } - - withoutDivider := bytes.TrimSpace(append(c[:startDivider], c[endDivider:]...)) - var ( - summary []byte - ) + var summary []byte if len(withoutDivider) > 0 { - summary = bytes.TrimSpace(withoutDivider[:endSummary]) + summary = bytes.TrimSpace(withoutDivider[:start]) } if addDiv { @@ -793,7 +785,7 @@ func splitUserDefinedSummaryAndContent(markup string, c []byte) (sc *summaryCont sc = &summaryContent{ summary: summary, - content: withoutDivider, + content: bytes.TrimSpace(withoutDivider), } return diff --git a/hugolib/page_content.go b/hugolib/page_content.go index dab46411e..5a8258279 100644 --- a/hugolib/page_content.go +++ b/hugolib/page_content.go @@ -27,7 +27,9 @@ import ( ) var ( - internalSummaryDivider = []byte("HUGOMORE42") + internalSummaryDividerBase = "HUGOMORE42" + internalSummaryDividerBaseBytes = []byte(internalSummaryDividerBase) + internalSummaryDividerPre = []byte("\n\n" + internalSummaryDividerBase + "\n\n") ) // The content related items on a Page. @@ -111,7 +113,7 @@ Loop: } case it.Type == pageparser.TypeLeadSummaryDivider: - result.Write(internalSummaryDivider) + result.Write(internalSummaryDividerPre) p.source.hasSummaryDivider = true // Need to determine if the page is truncated. f := func(item pageparser.Item) bool { diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 4499a97e0..18be64cee 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -483,6 +483,7 @@ func checkPageContent(t *testing.T, page *Page, content string, msg ...interface a := normalizeContent(content) b := normalizeContent(string(page.content())) if a != b { + t.Log(trace()) t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg) } } @@ -553,6 +554,7 @@ func normalizeExpected(ext, str string) string { } expected += fmt.Sprintf("Summary Same LineHUGOMORE42
- -Some more text
`, "Summary Same Line
", "Summary Same Line
\n\nSome more text
"}, - {"asciidoc", `sn
HUGOMORE42Some more text
sn
sn
Some more text
Summary Next Line
HUGOMORE42Some more text
Summary Next Line
Summary Next Line
Some more text
a
b
HUGOMORE42c
", "a
b
", "a
b
c
"}, - {"markdown", "a
b
cHUGOMORE42
", "a
b
c
", "a
b
c
"}, - {"markdown", "a
bHUGOMORE42
c
", "a
b
", "a
b
c
"}, - {"markdown", "aHUGOMORE42
b
c
", "a
", "a
b
c
"}, - {"markdown", " HUGOMORE42 ", "", ""}, - {"markdown", "HUGOMORE42", "", ""}, - {"markdown", "HUGOMORE42", "
", "
"}, - {"markdown", "HUGOMORE42
", "", "
"}, - {"markdown", "\n\n
HUGOMORE42
\n", "", ""}, - // Issue #2586 - // Note: Hugo will not split mid-sentence but will look for the closest - // paragraph end marker. This may be a change from Hugo 0.16, but it makes sense. - {"markdown", `this is an example HUGOMORE42of the issue.
`, - "this is an example of the issue.
", - "this is an example of the issue.
"}, - // Issue: #2538 - {"markdown", fmt.Sprintf(`%s
HUGOMORE42%s
-`, - strings.Repeat("A", 10), strings.Repeat("B", 31)), - fmt.Sprintf(`%s
`, strings.Repeat("A", 10)), - fmt.Sprintf(`%s
%s
`, strings.Repeat("A", 10), strings.Repeat("B", 31)), - }, - } { - - sc, err := splitUserDefinedSummaryAndContent(this.markup, []byte(this.content)) - - require.NoError(t, err) - require.NotNil(t, sc, fmt.Sprintf("[%d] Nil %s", i, this.markup)) - require.Equal(t, this.expectedSummary, string(sc.summary), fmt.Sprintf("[%d] Summary markup %s", i, this.markup)) - require.Equal(t, this.expectedContent, string(sc.content), fmt.Sprintf("[%d] Content markup %s", i, this.markup)) - } -} - func TestPageWithDelimiter(t *testing.T) { t.Parallel() assertFunc := func(t *testing.T, ext string, pages Pages) { @@ -720,11 +670,14 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { p := s.RegularPages[0] - if p.Summary() != template.HTML("The best static site generator.1\n
") { + if p.Summary() != template.HTML( + "The best static site generator.1
") { t.Fatalf("Got summary:\n%q", p.Summary()) } - if p.content() != template.HTML("The best static site generator.1\n
\nThe best static site generator.1
\n\nThis is a a shortcode.
:END", + "CONTENT:This is a a shortcode.
\n\nContent.
\n", + ) + + b.AssertFileContent("public/page-md-shortcode-same-line/index.html", + "SUMMARY:This is a a shortcode.
:END", + "CONTENT:This is a a shortcode.
\n\nSame line.
\n", + ) + + b.AssertFileContent("public/page-org-shortcode/index.html", + "SUMMARY:This is a a shortcode.
:END", + "CONTENT:This is a a shortcode.
\n\nContent.\t
\n", + ) + b.AssertFileContent("public/page-org-variant1/index.html", + "SUMMARY:Summary.
:END", + "CONTENT:Summary.
\n\nContent.\t
\n", + ) +} + // TODO(bep) this may be useful for other tests. func compareObjects(a interface{}, b interface{}) bool { aStr := strings.Split(fmt.Sprintf("%v", a), "") @@ -1705,6 +1722,8 @@ Len PlainWords: {{ len .PlainWords }} Truncated: {{ .Truncated }} Len Summary: {{ len .Summary }} Len Content: {{ len .Content }} + +SUMMARY:{{ .Summary }}:{{ len .Summary }}:END `} b := newTestSitesBuilder(t) @@ -1776,10 +1795,10 @@ Summary: In Chinese, 好 means good. b.AssertFileContent("public/p1/index.html", "WordCount: 510\nFuzzyWordCount: 600\nReadingTime: 3\nLen Plain: 2550\nLen PlainWords: 510\nTruncated: false\nLen Summary: 2549\nLen Content: 2557") - b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1570\nLen PlainWords: 314\nTruncated: true\nLen Summary: 34\nLen Content: 1592") + b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1569\nLen PlainWords: 314\nTruncated: true\nLen Summary: 25\nLen Content: 1583") - b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 639\nLen PlainWords: 7\nTruncated: true\nLen Summary: 52\nLen Content: 661") - b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 639\nLen PlainWords: 7\nTruncated: true\nLen Summary: 52\nLen Content: 661") + b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652") + b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652") b.AssertFileContent("public/p5/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 229\nLen Content: 653") b.AssertFileContent("public/p6/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: false\nLen Summary: 637\nLen Content: 653") diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 469cfeb4a..f80f40c48 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -4,7 +4,9 @@ import ( "io/ioutil" "path/filepath" "runtime" + "strconv" "testing" + "unicode/utf8" "bytes" "fmt" @@ -698,6 +700,26 @@ func dumpPages(pages ...*Page) { } } +func printStringIndexes(s string) { + lines := strings.Split(s, "\n") + i := 0 + + for _, line := range lines { + + for _, r := range line { + fmt.Printf("%-3s", strconv.Itoa(i)) + i += utf8.RuneLen(r) + } + i++ + fmt.Println() + for _, r := range line { + fmt.Printf("%-3s", string(r)) + } + fmt.Println() + + } + +} func isCI() bool { return os.Getenv("CI") != "" } diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index ddf109b3d..565be2994 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -194,6 +194,16 @@ func (l *pageLexer) consumeCRLF() bool { return consumed } +func (l *pageLexer) consumeSpace() { + for { + r := l.next() + if r == eof || !unicode.IsSpace(r) { + l.backup() + return + } + } +} + func lexMainSection(l *pageLexer) stateFunc { // Fast forward as far as possible. var l1, l2 int @@ -234,6 +244,8 @@ func lexMainSection(l *pageLexer) stateFunc { } l.summaryDividerChecked = true l.pos += len(l.summaryDivider) + // This makes it a little easier to reason about later. + l.consumeSpace() l.emit(TypeLeadSummaryDivider) } } diff --git a/parser/pageparser/pageparser_intro_test.go b/parser/pageparser/pageparser_intro_test.go index ba4a2c84b..60c431c10 100644 --- a/parser/pageparser/pageparser_intro_test.go +++ b/parser/pageparser/pageparser_intro_test.go @@ -37,7 +37,7 @@ var ( tstFrontMatterYAMLCRLF = nti(TypeFrontMatterYAML, "foo: \"bar\"\r\n") tstFrontMatterJSON = nti(TypeFrontMatterJSON, tstJSON+"\r\n") tstSomeText = nti(tText, "\nSome text.\n") - tstSummaryDivider = nti(TypeLeadSummaryDivider, "") + tstSummaryDivider = nti(TypeLeadSummaryDivider, "\n") tstHtmlStart = nti(TypeHTMLStart, "<") tstORG = ` @@ -65,8 +65,9 @@ var frontMatterTests = []lexerTest{ {"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}}, {"JSON front matter", tstJSON + "\r\n\nSome text.\n", []Item{tstFrontMatterJSON, tstSomeText, tstEOF}}, {"ORG front matter", tstORG + "\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, tstEOF}}, - {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more"), tstSomeText, tstEOF}}, - {"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, tstSomeText, tstEOF}}, + {"Summary divider ORG", tstORG + "\nSome text.\n# more\nSome text.\n", []Item{tstFrontMatterORG, tstSomeText, nti(TypeLeadSummaryDivider, "# more\n"), nti(tText, "Some text.\n"), tstEOF}}, + {"Summary divider", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstSummaryDivider, nti(tText, "Some text.\n"), tstEOF}}, + {"Summary divider same line", "+++\nfoo = \"bar\"\n+++\n\nSome text.Some text.\n", []Item{tstFrontMatterTOML, nti(tText, "\nSome text."), nti(TypeLeadSummaryDivider, ""), nti(tText, "Some text.\n"), tstEOF}}, } func TestFrontMatter(t *testing.T) {