From 90b257812ba798c22eb770e5074e39b57f307959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 27 Nov 2019 13:42:36 +0100 Subject: [PATCH 01/19] Add render template hooks for links and images This commit also revises the change detection for templates used by content files in server mode. Fixes #6545 Fixes #4663 Closes #6043 --- deps/deps.go | 4 + docs/content/en/content-management/formats.md | 8 +- .../content/en/getting-started/quick-start.md | 3 + .../_default/_markup/render-image.html | 2 + .../layouts/_default/_markup/render-link.html | 1 + docs/layouts/partials/deleteme.html | 1 + docs/layouts/shortcodes/deleteme.html | 3 + helpers/content.go | 4 +- helpers/general_test.go | 3 +- hugolib/content_render_hooks_test.go | 130 +++++++++++ hugolib/filesystems/basefs.go | 49 ++++- hugolib/hugo_modules_test.go | 3 + hugolib/hugo_sites.go | 54 ++++- hugolib/hugo_sites_build.go | 2 +- hugolib/page.go | 103 ++++++++- hugolib/page__meta.go | 16 +- hugolib/page__new.go | 23 +- hugolib/page__output.go | 42 ++-- hugolib/page__per_output.go | 45 +++- hugolib/page_test.go | 7 +- hugolib/page_unwrap_test.go | 1 + hugolib/pagecollections.go | 10 - hugolib/shortcode_page.go | 19 ++ hugolib/site.go | 122 ++++++----- hugolib/testhelpers_test.go | 9 +- identity/identity.go | 128 +++++++++++ identity/identity_test.go | 42 ++++ markup/asciidoc/convert.go | 5 + markup/blackfriday/convert.go | 5 + markup/converter/converter.go | 13 +- markup/converter/hooks/hooks.go | 46 ++++ markup/goldmark/ast_hooks.go | 77 +++++++ markup/goldmark/convert.go | 114 ++++++++-- markup/goldmark/convert_test.go | 15 +- markup/goldmark/render_link.go | 204 ++++++++++++++++++ markup/goldmark/toc.go | 113 +++++----- markup/mmark/convert.go | 5 + markup/org/convert.go | 6 + markup/pandoc/convert.go | 5 + markup/rst/convert.go | 5 + output/layout.go | 38 +++- output/layout_test.go | 2 + public/categories/index.xml | 13 ++ public/index.xml | 13 ++ public/sitemap.xml | 17 ++ public/tags/index.xml | 13 ++ tpl/template_info.go | 6 + tpl/tplimpl/ace.go | 5 +- tpl/tplimpl/template.go | 96 ++++++--- tpl/tplimpl/template_ast_transformers.go | 111 +++++++--- tpl/tplimpl/template_ast_transformers_test.go | 30 ++- tpl/tplimpl/template_funcs_test.go | 1 + 52 files changed, 1484 insertions(+), 308 deletions(-) create mode 100644 docs/layouts/_default/_markup/render-image.html create mode 100644 docs/layouts/_default/_markup/render-link.html create mode 100644 docs/layouts/partials/deleteme.html create mode 100644 docs/layouts/shortcodes/deleteme.html create mode 100644 hugolib/content_render_hooks_test.go create mode 100644 identity/identity.go create mode 100644 identity/identity_test.go create mode 100644 markup/converter/hooks/hooks.go create mode 100644 markup/goldmark/ast_hooks.go create mode 100644 markup/goldmark/render_link.go create mode 100644 public/categories/index.xml create mode 100644 public/index.xml create mode 100644 public/sitemap.xml create mode 100644 public/tags/index.xml diff --git a/deps/deps.go b/deps/deps.go index d7b381ce92e..4dc170a9713 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -282,6 +282,10 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er return nil, err } + if err != nil { + return nil, err + } + d.Site = cfg.Site // The resource cache is global so reuse. diff --git a/docs/content/en/content-management/formats.md b/docs/content/en/content-management/formats.md index ea056861657..3704ed8b071 100644 --- a/docs/content/en/content-management/formats.md +++ b/docs/content/en/content-management/formats.md @@ -23,7 +23,13 @@ You can put any file type into your `/content` directories, but Hugo uses the `m * [Shortcodes](/content-management/shortcodes/) processed * Layout applied -## List of content formats +{{< deleteme >}} + + +## List of content formats. + + + The current list of content formats in Hugo: diff --git a/docs/content/en/getting-started/quick-start.md b/docs/content/en/getting-started/quick-start.md index 143dc0a4172..2cabdfbb756 100644 --- a/docs/content/en/getting-started/quick-start.md +++ b/docs/content/en/getting-started/quick-start.md @@ -24,8 +24,11 @@ This quick start uses `macOS` in the examples. For instructions about how to ins It is recommended to have [Git](https://git-scm.com/downloads) installed to run this tutorial. {{% /note %}} +![Drag Racing](/images/Dragster.jpg "image title") +![Drag Racing](/images/Dragster2.jpg "image title") + ## Step 1: Install Hugo {{% note %}} diff --git a/docs/layouts/_default/_markup/render-image.html b/docs/layouts/_default/_markup/render-image.html new file mode 100644 index 00000000000..f4b9661dfc8 --- /dev/null +++ b/docs/layouts/_default/_markup/render-image.html @@ -0,0 +1,2 @@ +{{ $url := "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6a/Mating_pair_of_Castalius_rosimon_WLB_DSC_2823.jpg/1024px-Mating_pair_of_Castalius_rosimon_WLB_DSC_2823.jpg" | safeURL }} + \ No newline at end of file diff --git a/docs/layouts/_default/_markup/render-link.html b/docs/layouts/_default/_markup/render-link.html new file mode 100644 index 00000000000..0df3929f6b6 --- /dev/null +++ b/docs/layouts/_default/_markup/render-link.html @@ -0,0 +1 @@ +๐Ÿ˜‰๐Ÿ˜‰{{ .Text | safeHTML }} ๐Ÿ˜‰๐Ÿ˜‰ \ No newline at end of file diff --git a/docs/layouts/partials/deleteme.html b/docs/layouts/partials/deleteme.html new file mode 100644 index 00000000000..e4df2ce650f --- /dev/null +++ b/docs/layouts/partials/deleteme.html @@ -0,0 +1 @@ +THIS IS PARTIAL!!! \ No newline at end of file diff --git a/docs/layouts/shortcodes/deleteme.html b/docs/layouts/shortcodes/deleteme.html new file mode 100644 index 00000000000..b85b58fc1fc --- /dev/null +++ b/docs/layouts/shortcodes/deleteme.html @@ -0,0 +1,3 @@ +DELETEME PARTIAL: + +{{ partial "deleteme" }} \ No newline at end of file diff --git a/helpers/content.go b/helpers/content.go index 4dc4cd413bd..1c780fefe1b 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -25,13 +25,14 @@ import ( "github.com/gohugoio/hugo/common/loggers" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" "strings" ) @@ -78,6 +79,7 @@ func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero ContentFs: contentFs, Logger: logger, }) + if err != nil { return nil, err } diff --git a/helpers/general_test.go b/helpers/general_test.go index 104a4c35def..b45fb0e9b44 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -21,9 +21,8 @@ import ( "github.com/spf13/viper" - "github.com/gohugoio/hugo/common/loggers" - qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" ) diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go new file mode 100644 index 00000000000..74b054d757a --- /dev/null +++ b/hugolib/content_render_hooks_test.go @@ -0,0 +1,130 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import "testing" + +func TestRenderHooks(t *testing.T) { + // TODO1 markdownify + config := ` +baseURL="https://example.org" +workingDir="/mywork" +` + b := newTestSitesBuilder(t).WithWorkingDir("/mywork").WithConfigFile("toml", config).Running() + b.WithTemplatesAdded("_default/single.html", `{{ .Content }}`) + b.WithTemplatesAdded("shortcodes/myshortcode1.html", `{{ partial "mypartial1" }}`) + b.WithTemplatesAdded("shortcodes/myshortcode2.html", `{{ partial "mypartial2" }}`) + b.WithTemplatesAdded("shortcodes/myshortcode3.html", `SHORT3|`) + b.WithTemplatesAdded("shortcodes/myshortcode4.html", ` +
+{{ .Inner | markdownify }} +
+`) + b.WithTemplatesAdded("partials/mypartial1.html", `PARTIAL1`) + b.WithTemplatesAdded("partials/mypartial2.html", `PARTIAL2 {{ partial "mypartial3.html" }}`) + b.WithTemplatesAdded("partials/mypartial3.html", `PARTIAL3`) + b.WithTemplatesAdded("_default/_markup/render-link.html", `{{ with .Page }}{{ .Title }}{{ end }}|{{ .Destination | safeURL }}|Title: {{ .Title | safeHTML }}|Text: {{ .Text | safeHTML }}|END`) + b.WithTemplatesAdded("docs/_markup/render-link.html", `Link docs section: {{ .Text | safeHTML }}|END`) + b.WithTemplatesAdded("_default/_markup/render-image.html", `IMAGE: {{ .Page.Title }}||{{ .Destination | safeURL }}|Title: {{ .Title | safeHTML }}|Text: {{ .Text | safeHTML }}|END`) + + b.WithContent("blog/p1.md", `--- +title: Cool Page +--- + +[First Link](https://www.google.com "Google's Homepage") + +{{< myshortcode3 >}} + +[Second Link](https://www.google.com "Google's Homepage") + +Image: + +![Drag Racing](/images/Dragster.jpg "image title") + + +`, "blog/p2.md", `--- +title: Cool Page2 +layout: mylayout +--- + +{{< myshortcode1 >}} + +[Some Text](https://www.google.com "Google's Homepage") + + + +`, "blog/p3.md", `--- +title: Cool Page3 +--- + +{{< myshortcode2 >}} + + +`, "docs/docs1.md", `--- +title: Docs 1 +--- + + +[Docs 1](https://www.google.com "Google's Homepage") + + +`, "blog/p4.md", `--- +title: Cool Page With Image +--- + +Image: + +![Drag Racing](/images/Dragster.jpg "image title") + + +`, "blog/p5.md", `--- +title: Cool Page With Markdownify +--- + +{{< myshortcode4 >}} +Inner Link: [Inner Link](https://www.google.com "Google's Homepage") +{{< /myshortcode4 >}} + +`) + b.Build(BuildCfg{}) + b.AssertFileContent("public/blog/p1/index.html", ` +

Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END

+Text: Second +SHORT3| +

IMAGE: Cool Page||/images/Dragster.jpg|Title: image title|Text: Drag Racing|END

+`) + b.AssertFileContent("public/blog/p2/index.html", `PARTIAL`) + b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3`) + b.AssertFileContent("public/docs/docs1/index.html", `Link docs section: Docs 1|END`) + b.AssertFileContent("public/blog/p4/index.html", `

IMAGE: Cool Page With Image||/images/Dragster.jpg|Title: image title|Text: Drag Racing|END

`) + // The regular markdownify func currently gets regular links. + b.AssertFileContent("public/blog/p5/index.html", "Inner Link: Inner Link\n") + + b.EditFiles( + "layouts/_default/_markup/render-link.html", `EDITED: {{ .Destination | safeURL }}|`, + "layouts/_default/_markup/render-image.html", `IMAGE EDITED: {{ .Destination | safeURL }}|`, + "layouts/docs/_markup/render-link.html", `DOCS EDITED: {{ .Destination | safeURL }}|`, + "layouts/partials/mypartial1.html", `PARTIAL1_EDITED`, + "layouts/partials/mypartial3.html", `PARTIAL3_EDITED`, + "layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`, + ) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/blog/p1/index.html", `

EDITED: https://www.google.com|

`, "SHORT3_EDITED|") + b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`) + b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3_EDITED`) + b.AssertFileContent("public/docs/docs1/index.html", `DOCS EDITED: https://www.google.com|

`) + b.AssertFileContent("public/blog/p4/index.html", `IMAGE EDITED: /images/Dragster.jpg|`) + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index de6baa130d7..cdc39ce61cb 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -126,10 +126,28 @@ type SourceFilesystems struct { StaticDirs []hugofs.FileMetaInfo } +// FileSystems returns the FileSystems relevant for the change detection +// in server mode. +// Note: This does currently not return any static fs. +func (s *SourceFilesystems) FileSystems() []*SourceFilesystem { + return []*SourceFilesystem{ + s.Content, + s.Data, + s.I18n, + s.Layouts, + s.Archetypes, + // TODO(bep) static + } + +} + // A SourceFilesystem holds the filesystem for a given source type in Hugo (data, // i18n, layouts, static) and additional metadata to be able to use that filesystem // in server mode. type SourceFilesystem struct { + // Name matches one in files.ComponentFolders + Name string + // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs @@ -275,6 +293,19 @@ func (d *SourceFilesystem) Contains(filename string) bool { return false } +// Path returns the relative path to the given filename if it is a member of +// of the current filesystem, an empty string if not. +func (d *SourceFilesystem) Path(filename string) string { + for _, dir := range d.Dirs { + meta := dir.Meta() + if strings.HasPrefix(filename, meta.Filename()) { + p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) + return p + } + } + return "" +} + // RealDirs gets a list of absolute paths to directories starting from the given // path. func (d *SourceFilesystem) RealDirs(from string) []string { @@ -349,12 +380,14 @@ func newSourceFilesystemsBuilder(p *paths.Paths, logger *loggers.Logger, b *Base return &sourceFilesystemsBuilder{p: p, logger: logger, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}} } -func (b *sourceFilesystemsBuilder) newSourceFilesystem(fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { +func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs, dirs []hugofs.FileMetaInfo) *SourceFilesystem { return &SourceFilesystem{ + Name: name, Fs: fs, Dirs: dirs, } } + func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { if b.theBigFs == nil { @@ -369,12 +402,12 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { createView := func(componentID string) *SourceFilesystem { if b.theBigFs == nil || b.theBigFs.overlayMounts == nil { - return b.newSourceFilesystem(hugofs.NoOpFs, nil) + return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil) } dirs := b.theBigFs.overlayDirs[componentID] - return b.newSourceFilesystem(afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) + return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs) } @@ -392,14 +425,14 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return nil, err } - b.result.Data = b.newSourceFilesystem(dataFs, dataDirs) + b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs) i18nDirs := b.theBigFs.overlayDirs[files.ComponentFolderI18n] i18nFs, err := hugofs.NewSliceFs(i18nDirs...) if err != nil { return nil, err } - b.result.I18n = b.newSourceFilesystem(i18nFs, i18nDirs) + b.result.I18n = b.newSourceFilesystem(files.ComponentFolderI18n, i18nFs, i18nDirs) contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent] contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent) @@ -409,7 +442,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return nil, errors.Wrap(err, "create content filesystem") } - b.result.Content = b.newSourceFilesystem(contentFs, contentDirs) + b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs) b.result.Work = afero.NewReadOnlyFs(b.theBigFs.overlayFull) @@ -421,13 +454,13 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { if b.theBigFs.staticPerLanguage != nil { // Multihost mode for k, v := range b.theBigFs.staticPerLanguage { - sfs := b.newSourceFilesystem(v, b.result.StaticDirs) + sfs := b.newSourceFilesystem(files.ComponentFolderStatic, v, b.result.StaticDirs) sfs.PublishFolder = k ms[k] = sfs } } else { bfs := afero.NewBasePathFs(b.theBigFs.overlayMountsStatic, files.ComponentFolderStatic) - ms[""] = b.newSourceFilesystem(bfs, b.result.StaticDirs) + ms[""] = b.newSourceFilesystem(files.ComponentFolderStatic, bfs, b.result.StaticDirs) } return b.result, nil diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 9ba039c7430..ddc0ef59b46 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -40,6 +40,9 @@ import ( // TODO(bep) this fails when testmodBuilder is also building ... func TestHugoModules(t *testing.T) { + if !isCI() { + t.Skip("skip (relative) long running modules test when running locally") + } t.Parallel() if hugo.GoMinorVersion() < 12 { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index c0d75c09f52..479ff4bcd2f 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -20,6 +20,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/identity" + radix "github.com/armon/go-radix" "github.com/gohugoio/hugo/output" @@ -411,7 +413,6 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { } d.OutputFormatsConfig = s.outputFormatsConfig } - } return nil @@ -806,12 +807,55 @@ func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Page return h.Sites[0].findPagesByKindIn(kind, inPages) } -func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages { - var pages page.Pages +func (h *HugoSites) resetPageStateFromEvents(ids identity.Identities) { + idset := ids.ToIdentitySet() + hasIdentify := func(v interface{}) bool { + if id, ok := v.(identity.Provider); ok { + if idset[id.GetIdentity()] { + return true + } + } + if idp, ok := v.(identity.IdentitiesProvider); ok { + for id, _ := range idp.GetIdentities() { + if idset[id.GetIdentity()] { + return true + } + } + } + return false + } + for _, s := range h.Sites { - pages = append(pages, s.findPagesByShortcode(shortcode)...) + PAGES: + for _, p := range s.rawAllPages { + OUTPUTS: + for _, po := range p.pageOutputs { + if c := po.cp; c != nil { + if converted := c.convertedResult; converted != nil { + if hasIdentify(converted) { + c.Reset() + p.forceRender = true + continue OUTPUTS + } + } + } + } + + for _, s := range p.shortcodeState.shortcodes { + for _, id := range ids { + if s.info.Search(id.GetIdentity()) != nil { + for _, po := range p.pageOutputs { + if po.cp != nil { + po.cp.Reset() + } + } + p.forceRender = true + continue PAGES + } + } + } + } } - return pages } // Used in partial reloading to determine if the change is in a bundle. diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index a70a19e7c31..d749ff581d5 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -71,7 +71,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { if conf.whatChanged == nil { // Assume everything has changed - conf.whatChanged = &whatChanged{source: true, other: true} + conf.whatChanged = &whatChanged{source: true} } var prepareErr error diff --git a/hugolib/page.go b/hugolib/page.go index b0e8c4359fd..625206ad114 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -23,6 +23,10 @@ import ( "sort" "strings" + "github.com/gohugoio/hugo/tpl" + + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/common/maps" @@ -46,6 +50,7 @@ import ( "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" @@ -59,7 +64,11 @@ var ( var ( pageTypesProvider = resource.NewResourceTypesProvider(media.OctetType, pageResourceType) - nopPageOutput = &pageOutput{pagePerOutputProviders: nopPagePerOutput} + nopPageOutput = &pageOutput{ + pagePerOutputProviders: nopPagePerOutput, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + } ) // pageContext provides contextual information about this page, for error @@ -317,6 +326,98 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error { return nil } +func (ps *pageState) initOutputFormats() error { + + if len(ps.pageOutputs) == 0 { + return nil + } + + c := ps.getContentConverter() + if c == nil || !c.Supports(converter.FeatureRenderHooks) { + return nil + } + + templSet := make(map[identity.Identity]bool) + canReuse := true + + for _, o := range ps.pageOutputs { + if !o.render || o.cp == nil { + continue + } + h, err := ps.createRenderHooks(o.f) + if err != nil { + return err + } + if h == nil { + continue + } + if canReuse { + for _, r := range []hooks.LinkRenderer{h.LinkRenderer, h.ImageRenderer} { + if !canReuse { + break + } + if r != nil { + if len(templSet) != 0 { + // There may be a template per output format. + // In that case we need to re-render. + canReuse = templSet[r.GetIdentity()] + } + templSet[r.GetIdentity()] = true + } + } + } + o.cp.renderHooks = h + o.cp.renderHooksHaveVariants = !canReuse + } + + return nil +} + +func (p *pageState) createRenderHooks(f output.Format) (*hooks.Render, error) { + + layoutDescriptor := p.getLayoutDescriptor() + layoutDescriptor.RenderingHook = true + layoutDescriptor.LayoutOverride = false + + layoutDescriptor.Kind = "render-link" + linkLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + if err != nil { + return nil, err + } + + layoutDescriptor.Kind = "render-image" + imageLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + if err != nil { + return nil, err + } + + if linkLayouts == nil && imageLayouts == nil { + return nil, nil + } + + var linkRenderer hooks.LinkRenderer + var imageRenderer hooks.LinkRenderer + + if templ, found := p.s.lookupTemplate(linkLayouts...); found { + linkRenderer = contentLinkRenderer{ + Provider: templ.(tpl.TemplateInfoProvider).TemplateInfo(), + templ: templ, + } + } + + if templ, found := p.s.lookupTemplate(imageLayouts...); found { + imageRenderer = contentLinkRenderer{ + Provider: templ.(tpl.TemplateInfoProvider).TemplateInfo(), + templ: templ, + } + } + + return &hooks.Render{ + LinkRenderer: linkRenderer, + ImageRenderer: imageRenderer, + }, nil +} + func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor { p.layoutDescriptorInit.Do(func() { var section string diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index 1fc69c21826..37e333cb40b 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -592,7 +592,7 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte return nil } -func (p *pageMeta) applyDefaultValues() error { +func (p *pageMeta) applyDefaultValues(ps *pageState) error { if p.markup == "" { if !p.File().IsZero() { // Fall back to file extension @@ -656,15 +656,19 @@ func (p *pageMeta) applyDefaultValues() error { return errors.Errorf("no content renderer found for markup %q", p.markup) } - cpp, err := cp.New(converter.DocumentContext{ - DocumentID: p.f.UniqueID(), - DocumentName: p.f.Path(), - ConfigOverrides: renderingConfigOverrides, - }) + cpp, err := cp.New( + converter.DocumentContext{ + Document: newPageForRenderHook(ps), + DocumentID: p.f.UniqueID(), + DocumentName: p.f.Path(), + ConfigOverrides: renderingConfigOverrides, + }, + ) if err != nil { return err } + p.contentConverter = cpp } diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 99bf305aa58..37db148f8bd 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -112,7 +112,7 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page } } - if err := metaProvider.applyDefaultValues(); err != nil { + if err := metaProvider.applyDefaultValues(ps); err != nil { return err } @@ -134,7 +134,7 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page } makeOut := func(f output.Format, render bool) *pageOutput { - return newPageOutput(nil, ps, pp, f, render) + return newPageOutput(ps, pp, f, render) } if ps.m.standalone { @@ -158,6 +158,10 @@ func newPageFromMeta(meta map[string]interface{}, metaProvider *pageMeta) (*page return nil, err } + if err := ps.initOutputFormats(); err != nil { + return nil, err + } + return nil, nil }) @@ -234,7 +238,7 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope return ps.wrapError(err) } - if err := metaProvider.applyDefaultValues(); err != nil { + if err := metaProvider.applyDefaultValues(ps); err != nil { return err } @@ -264,18 +268,17 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope } _, render := outputFormatsForPage.GetByName(f.Name) - var contentProvider *pageContentOutput + po := newPageOutput(ps, pp, f, render) if reuseContent && i > 0 { - contentProvider = ps.pageOutputs[0].cp + po.initContentProvider(ps.pageOutputs[0].cp) } else { - var err error - contentProvider, err = contentPerOutput(f) + contentProvider, err := contentPerOutput(po) if err != nil { return nil, err } + po.initContentProvider(contentProvider) } - po := newPageOutput(contentProvider, ps, pp, f, render) ps.pageOutputs[i] = po created[f.Name] = po } @@ -284,6 +287,10 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope return nil, err } + if err := ps.initOutputFormats(); err != nil { + return nil, err + } + return nil, nil }) diff --git a/hugolib/page__output.go b/hugolib/page__output.go index 764c46a937b..b07dbc9126d 100644 --- a/hugolib/page__output.go +++ b/hugolib/page__output.go @@ -20,7 +20,6 @@ import ( ) func newPageOutput( - cp *pageContentOutput, // may be nil ps *pageState, pp pagePaths, f output.Format, @@ -45,36 +44,23 @@ func newPageOutput( paginatorProvider = pag } - var ( - contentProvider page.ContentProvider = page.NopPage - tableOfContentsProvider page.TableOfContentsProvider = page.NopPage - ) - - if cp != nil { - contentProvider = cp - tableOfContentsProvider = cp - } - providers := struct { - page.ContentProvider - page.TableOfContentsProvider page.PaginatorProvider resource.ResourceLinksProvider targetPather }{ - contentProvider, - tableOfContentsProvider, paginatorProvider, linksProvider, targetPathsProvider, } po := &pageOutput{ - f: f, - cp: cp, - pagePerOutputProviders: providers, - render: render, - paginator: pag, + f: f, + pagePerOutputProviders: providers, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + render: render, + paginator: pag, } return po @@ -94,16 +80,28 @@ type pageOutput struct { // used in template(s). paginator *pagePaginator - // This interface provides the functionality that is specific for this + // These interface provides the functionality that is specific for this // output format. pagePerOutputProviders + page.ContentProvider + page.TableOfContentsProvider - // This may be nil. + // May be nil. cp *pageContentOutput } +func (p *pageOutput) initContentProvider(cp *pageContentOutput) { + if cp == nil { + return + } + p.ContentProvider = cp + p.TableOfContentsProvider = cp + p.cp = cp +} + func (p *pageOutput) enablePlaceholders() { if p.cp != nil { p.cp.enablePlaceholders() } + } diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index bc2a0accc04..a9639c04747 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -23,6 +23,8 @@ import ( "sync" "unicode/utf8" + "github.com/gohugoio/hugo/markup/converter/hooks" + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/lazy" @@ -58,14 +60,14 @@ var ( } ) -func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutput, error) { +func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput, error) { parent := p.init - return func(f output.Format) (*pageContentOutput, error) { + return func(po *pageOutput) (*pageContentOutput, error) { cp := &pageContentOutput{ p: p, - f: f, + f: po.f, } initContent := func() (err error) { @@ -83,13 +85,22 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu var hasVariants bool + f := po.f cp.contentPlaceholders, hasVariants, err = p.shortcodeState.renderShortcodesForPage(p, f) if err != nil { return err } - if p.render && !hasVariants { - // We can reuse this for the other output formats + enableReuse := po.render && !hasVariants + enableReuse = enableReuse && !cp.renderHooksHaveVariants + + if enableReuse { + // Reuse this for the other output formats. + // We may improve on this, but we really want to avoid re-rendering the content + // to all output formats. + // The current rule is that if you need output format-aware shortcodes or + // content rendering hooks, create a output format-specific template, e.g. + // myshortcode.amp.html. cp.enableReuse() } @@ -178,6 +189,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu // Recursive loops can only happen in content files with template code (shortcodes etc.) // Avoid creating new goroutines if we don't have to. needTimeout := !p.renderable || p.shortcodeState.hasShortcodes() + needTimeout = needTimeout || cp.renderHooks != nil if needTimeout { cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) { @@ -211,7 +223,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu type pageContentOutput struct { f output.Format - // If we can safely reuse this for other output formats. + // If we can reuse this for other output formats. reuse bool reuseInit sync.Once @@ -224,6 +236,11 @@ type pageContentOutput struct { placeholdersEnabled bool placeholdersEnabledInit sync.Once + // May be nil. + renderHooks *hooks.Render + // Set if there are more than one output format variant + renderHooksHaveVariants bool + // Content state workContent []byte @@ -248,6 +265,12 @@ type pageContentOutput struct { readingTime int } +func (p *pageContentOutput) Reset() { + p.p.initOutputFormats() + p.initMain.Reset() + p.initPlain.Reset() +} + func (p *pageContentOutput) Content() (interface{}, error) { if p.p.s.initInit(p.initMain, p.p) { return p.content, nil @@ -332,10 +355,12 @@ func (p *pageContentOutput) setAutoSummary() error { } func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) { - return cp.p.getContentConverter().Convert( + c := cp.p.getContentConverter() + return c.Convert( converter.RenderContext{ - Src: content, - RenderTOC: true, + Src: content, + RenderTOC: true, + RenderHooks: cp.renderHooks, }) } @@ -392,9 +417,7 @@ func (p *pageContentOutput) enableReuse() { // these will be shifted out when rendering a given output format. type pagePerOutputProviders interface { targetPather - page.ContentProvider page.PaginatorProvider - page.TableOfContentsProvider resource.ResourceLinksProvider } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index dc8bc821c15..f4bf3ac0040 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -93,12 +93,6 @@ Summary Next Line. {{
}}. More text here. Some more text -` - - simplePageWithEmbeddedScript = `--- -title: Simple ---- - ` simplePageWithSummaryDelimiterSameLine = `--- @@ -325,6 +319,7 @@ func normalizeContent(c string) string { } func checkPageTOC(t *testing.T, page page.Page, toc string) { + t.Helper() if page.TableOfContents() != template.HTML(toc) { t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc) } diff --git a/hugolib/page_unwrap_test.go b/hugolib/page_unwrap_test.go index 20888166ad7..bcc1b769a4f 100644 --- a/hugolib/page_unwrap_test.go +++ b/hugolib/page_unwrap_test.go @@ -26,6 +26,7 @@ func TestUnwrapPage(t *testing.T) { p := &pageState{} c.Assert(mustUnwrap(newPageForShortcode(p)), qt.Equals, p) + c.Assert(mustUnwrap(newPageForRenderHook(p)), qt.Equals, p) } func mustUnwrap(v interface{}) page.Page { diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index 7e9682e90e1..adcbbccefef 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -358,16 +358,6 @@ func (c *PageCollections) removePage(page *pageState) { } } -func (c *PageCollections) findPagesByShortcode(shortcode string) page.Pages { - var pages page.Pages - for _, p := range c.rawAllPages { - if p.HasShortcode(shortcode) { - pages = append(pages, p) - } - } - return pages -} - func (c *PageCollections) replacePage(page *pageState) { // will find existing page that matches filepath and remove it c.removePage(page) diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go index e8a3a37e19b..5a56e434f2f 100644 --- a/hugolib/shortcode_page.go +++ b/hugolib/shortcode_page.go @@ -54,3 +54,22 @@ func (p *pageForShortcode) TableOfContents() template.HTML { p.p.enablePlaceholders() return p.toc } + +// This is what is sent into the content render hooks (link, image). +type pageForRenderHooks struct { + page.PageWithoutContent + page.TableOfContentsProvider + page.ContentProvider +} + +func newPageForRenderHook(p *pageState) page.Page { + return &pageForRenderHooks{ + PageWithoutContent: p, + ContentProvider: page.NopPage, + TableOfContentsProvider: page.NopPage, + } +} + +func (p *pageForRenderHooks) page() page.Page { + return p.PageWithoutContent.(page.Page) +} diff --git a/hugolib/site.go b/hugolib/site.go index 1df7d6076db..be1d178ba6d 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -28,6 +28,10 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/identity" + + "github.com/gohugoio/hugo/markup/converter/hooks" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/markup/converter" @@ -801,7 +805,6 @@ func (s *Site) multilingual() *Multilingual { type whatChanged struct { source bool - other bool files map[string]bool } @@ -888,10 +891,11 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event { // It returns whetever the content source was changed. // TODO(bep) clean up/rewrite this method. func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error { - events = s.filterFileEvents(events) events = s.translateFileEvents(events) + var changeIdentities identity.Identities + s.Log.DEBUG.Printf("Rebuild for events %q", events) h := s.h @@ -902,11 +906,12 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro sourceChanged = []fsnotify.Event{} sourceReallyChanged = []fsnotify.Event{} contentFilesChanged []string - tmplChanged = []fsnotify.Event{} - dataChanged = []fsnotify.Event{} - i18nChanged = []fsnotify.Event{} - shortcodesChanged = make(map[string]bool) - sourceFilesChanged = make(map[string]bool) + + tmplChanged bool + dataChanged bool + i18nChanged bool + + sourceFilesChanged = make(map[string]bool) // prevent spamming the log on changes logger = helpers.NewDistinctFeedbackLogger() @@ -915,37 +920,34 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro var cachePartitions []string for _, ev := range events { - if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) - } + id, found := s.eventToIdentity(ev) + if found { + changeIdentities = append(changeIdentities, id) + + if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { + cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) + } + + switch id.Type { + case files.ComponentFolderContent: + logger.Println("Source changed", ev) + sourceChanged = append(sourceChanged, ev) + case files.ComponentFolderLayouts: + logger.Println("Template changed", ev) + tmplChanged = true + case files.ComponentFolderData: + logger.Println("Data changed", ev) + dataChanged = true + case files.ComponentFolderI18n: + logger.Println("i18n changed", ev) + i18nChanged = true - if s.isContentDirEvent(ev) { - logger.Println("Source changed", ev) - sourceChanged = append(sourceChanged, ev) - } - if s.isLayoutDirEvent(ev) { - logger.Println("Template changed", ev) - tmplChanged = append(tmplChanged, ev) - - if strings.Contains(ev.Name, "shortcodes") { - shortcode := filepath.Base(ev.Name) - shortcode = strings.TrimSuffix(shortcode, filepath.Ext(shortcode)) - shortcodesChanged[shortcode] = true } - } - if s.isDataDirEvent(ev) { - logger.Println("Data changed", ev) - dataChanged = append(dataChanged, ev) - } - if s.isI18nEvent(ev) { - logger.Println("i18n changed", ev) - i18nChanged = append(dataChanged, ev) } } changed := &whatChanged{ - source: len(sourceChanged) > 0 || len(shortcodesChanged) > 0, - other: len(tmplChanged) > 0 || len(i18nChanged) > 0 || len(dataChanged) > 0, + source: len(sourceChanged) > 0, files: sourceFilesChanged, } @@ -960,7 +962,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) } - if len(tmplChanged) > 0 || len(i18nChanged) > 0 { + if tmplChanged || i18nChanged { sites := s.h.Sites first := sites[0] @@ -989,7 +991,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro } } - if len(dataChanged) > 0 { + if dataChanged { s.h.init.data.Reset() } @@ -1018,18 +1020,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro sourceFilesChanged[ev.Name] = true } - for shortcode := range shortcodesChanged { - // There are certain scenarios that, when a shortcode changes, - // it isn't sufficient to just rerender the already parsed shortcode. - // One example is if the user adds a new shortcode to the content file first, - // and then creates the shortcode on the file system. - // To handle these scenarios, we must do a full reprocessing of the - // pages that keeps a reference to the changed shortcode. - pagesWithShortcode := h.findPagesByShortcode(shortcode) - for _, p := range pagesWithShortcode { - contentFilesChanged = append(contentFilesChanged, p.File().Filename()) - } - } + h.resetPageStateFromEvents(changeIdentities) if len(sourceReallyChanged) > 0 || len(contentFilesChanged) > 0 { var filenamesChanged []string @@ -1218,20 +1209,14 @@ func (s *Site) initializeSiteInfo() error { return nil } -func (s *Site) isI18nEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsI18n(e.Name) -} - -func (s *Site) isDataDirEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsData(e.Name) -} - -func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { - return s.BaseFs.SourceFilesystems.IsLayout(e.Name) -} +func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { + for _, fs := range s.BaseFs.SourceFilesystems.FileSystems() { + if p := fs.Path(e.Name); p != "" { + return identity.NewPathIdentity(fs.Name, p), true + } + } -func (s *Site) isContentDirEvent(e fsnotify.Event) bool { - return s.BaseFs.IsContent(e.Name) + return identity.PathIdentity{}, false } func (s *Site) readAndProcessContent(filenames ...string) error { @@ -1562,6 +1547,25 @@ var infoOnMissingLayout = map[string]bool{ "404": true, } +type contentLinkRenderer struct { + identity.Provider + templ tpl.Template +} + +func (r contentLinkRenderer) Render(w io.Writer, ctx hooks.LinkContext) error { + return r.templ.Execute(w, ctx) +} + +func (s *Site) lookupTemplate(layouts ...string) (tpl.Template, bool) { + for _, l := range layouts { + if templ, found := s.Tmpl.Lookup(l); found { + return templ, true + } + } + + return nil, false +} + func (s *Site) renderForLayouts(name, outputFormat string, d interface{}, w io.Writer, layouts ...string) (err error) { templ := s.findFirstTemplate(layouts...) if templ == nil { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index d861a5e0941..9912af65b40 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -389,8 +389,9 @@ func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder { var changedFiles []string for i := 0; i < len(filenameContent); i += 2 { filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] - changedFiles = append(changedFiles, filename) - writeSource(s.T, s.Fs, s.absFilename(filename), content) + absFilename := s.absFilename(filename) + changedFiles = append(changedFiles, absFilename) + writeSource(s.T, s.Fs, absFilename, content) } s.changedFiles = changedFiles @@ -963,10 +964,6 @@ func isCI() bool { return os.Getenv("CI") != "" } -func isGo111() bool { - return strings.Contains(runtime.Version(), "1.11") -} - // See https://github.com/golang/go/issues/19280 // Not in use. var parallelEnabled = true diff --git a/identity/identity.go b/identity/identity.go new file mode 100644 index 00000000000..d7be4ebba66 --- /dev/null +++ b/identity/identity.go @@ -0,0 +1,128 @@ +package identity + +import ( + "path/filepath" + "strings" + "sync" +) + +// NewIdentityManager creates a new Manager starting at root. +func NewIdentityManager(root Provider) Manager { + return &identityManager{ + Provider: root, + children: make(Identities, 0), + } +} + +// NewPathIdentity creates a new Identity with the two identifiers +// type and path. +func NewPathIdentity(typ, path string) PathIdentity { + path = strings.TrimPrefix(filepath.ToSlash(path), "/") + return PathIdentity{Type: typ, Path: path} +} + +// Identities stores identity providers. +type Identities []Provider + +// A set of identities. +type IdentitiesSet map[Identity]bool + +// ToIdentitySet creates a set of these Identities. +func (ids Identities) ToIdentitySet() map[Identity]bool { + m := make(map[Identity]bool) + for _, id := range ids { + m[id.GetIdentity()] = true + } + return m +} + +func (ids Identities) search(id Identity) Provider { + for _, v := range ids { + vid := v.GetIdentity() + + if vid == id { + return v + } + + if idsp, ok := v.(ChildIdentitiesProvider); ok { + if nested := idsp.GetChildIdentities().search(id); nested != nil { + return nested + } + } + } + + return nil +} + +// IdentitiesProvider provides Identities as a set. +type IdentitiesProvider interface { + GetIdentities() IdentitiesSet +} + +// ChildIdentitiesProvider provides child Identities. +type ChildIdentitiesProvider interface { + GetChildIdentities() Identities +} + +// Identity represents an thing that can provide an identify. This can be +// any Go type, but the Identity returned by GetIdentify must be hashable. +type Identity interface { + Provider + Name() string +} + +// Manager manages identities, and is itself a Provider of Identity. +type Manager interface { + ChildIdentitiesProvider + Provider + Add(ids ...Provider) + Search(id Identity) Provider +} + +// A PathIdentity is a common identity identified by a type and a path, e.g. "layouts" and "_default/single.html". +type PathIdentity struct { + Type string + Path string +} + +// GetIdentity returns itself. +func (id PathIdentity) GetIdentity() Identity { + return id +} + +// Name returns the Path. +func (id PathIdentity) Name() string { + return id.Path +} + +// Provider provides the hashable Identity. +type Provider interface { + GetIdentity() Identity +} + +type identityManager struct { + sync.RWMutex + Provider + children Identities +} + +func (im *identityManager) Add(ids ...Provider) { + im.Lock() + im.children = append(im.children, ids...) + im.Unlock() +} + +func (im *identityManager) GetChildIdentities() Identities { + return im.children +} + +func (im *identityManager) Search(id Identity) Provider { + im.RLock() + defer im.RUnlock() + if id == im.GetIdentity() { + return im + } + v := im.children.search(id) + + return v +} diff --git a/identity/identity_test.go b/identity/identity_test.go new file mode 100644 index 00000000000..78e7a3b5e15 --- /dev/null +++ b/identity/identity_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package identity + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestIdentityManager(t *testing.T) { + c := qt.New(t) + + id1 := testIdentity{name: "id1"} + im := NewIdentityManager(id1) + + c.Assert(im.Search(id1).GetIdentity(), qt.Equals, id1) + c.Assert(im.Search(testIdentity{name: "notfound"}), qt.Equals, nil) +} + +type testIdentity struct { + name string +} + +func (id testIdentity) GetIdentity() Identity { + return id +} + +func (id testIdentity) Name() string { + return id.name +} diff --git a/markup/asciidoc/convert.go b/markup/asciidoc/convert.go index 65fdde0f564..a72aac39198 100644 --- a/markup/asciidoc/convert.go +++ b/markup/asciidoc/convert.go @@ -18,6 +18,7 @@ package asciidoc import ( "os/exec" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -47,6 +48,10 @@ func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Resu return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil } +func (c *asciidocConverter) Supports(feature identity.Identity) bool { + return false +} + // getAsciidocContent calls asciidoctor or asciidoc as an external helper // to convert AsciiDoc content to HTML. func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte { diff --git a/markup/blackfriday/convert.go b/markup/blackfriday/convert.go index 350defcb63c..3df23c7ae74 100644 --- a/markup/blackfriday/convert.go +++ b/markup/blackfriday/convert.go @@ -15,6 +15,7 @@ package blackfriday import ( + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config" "github.com/gohugoio/hugo/markup/converter" "github.com/russross/blackfriday" @@ -72,6 +73,10 @@ func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.R return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil } +func (c *blackfridayConverter) Supports(feature identity.Identity) bool { + return false +} + func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer { flags := getFlags(renderTOC, c.bf) diff --git a/markup/converter/converter.go b/markup/converter/converter.go index a1141f65ccc..dd7b4a347d8 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -16,6 +16,8 @@ package converter import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/spf13/afero" @@ -67,6 +69,7 @@ func (n newConverter) Name() string { // another format, e.g. Markdown to HTML. type Converter interface { Convert(ctx RenderContext) (Result, error) + Supports(feature identity.Identity) bool } // Result represents the minimum returned from Convert. @@ -94,6 +97,7 @@ func (b Bytes) Bytes() []byte { // DocumentContext holds contextual information about the document to convert. type DocumentContext struct { + Document interface{} // May be nil. Usually a page.Page DocumentID string DocumentName string ConfigOverrides map[string]interface{} @@ -101,6 +105,11 @@ type DocumentContext struct { // RenderContext holds contextual information about the content to render. type RenderContext struct { - Src []byte - RenderTOC bool + Src []byte + RenderTOC bool + RenderHooks *hooks.Render } + +var ( + FeatureRenderHooks = identity.NewPathIdentity("markup", "renderingHooks") +) diff --git a/markup/converter/hooks/hooks.go b/markup/converter/hooks/hooks.go new file mode 100644 index 00000000000..c938753a160 --- /dev/null +++ b/markup/converter/hooks/hooks.go @@ -0,0 +1,46 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hooks + +import ( + "io" + + "github.com/gohugoio/hugo/identity" +) + +type LinkContext interface { + Page() interface{} + Destination() string + Title() string + Text() string +} + +type Render struct { + LinkRenderer LinkRenderer + ImageRenderer LinkRenderer +} + +func (h *Render) GetChildIdentities() identity.Identities { + var ids identity.Identities + if h.LinkRenderer != nil { + ids = append(ids, h.LinkRenderer) + } + return ids + +} + +type LinkRenderer interface { + Render(w io.Writer, ctx LinkContext) error + identity.Provider +} diff --git a/markup/goldmark/ast_hooks.go b/markup/goldmark/ast_hooks.go new file mode 100644 index 00000000000..c289aad9af9 --- /dev/null +++ b/markup/goldmark/ast_hooks.go @@ -0,0 +1,77 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goldmark + +import ( + "github.com/gohugoio/hugo/markup/converter" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +func newASTExtension(cfg converter.ProviderConfig) goldmark.Extender { + return &astExtension{cfg: cfg} +} + +type astExtension struct { + cfg converter.ProviderConfig +} + +func (e *astExtension) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithASTTransformers(util.Prioritized(&astTransformer{cfg: e.cfg}, 10))) +} + +type astHook interface { + Done() + Visit(n ast.Node, entering bool) (ast.WalkStatus, error) +} + +type astTransformer struct { + cfg converter.ProviderConfig +} + +func (t *astTransformer) Transform(n *ast.Document, reader text.Reader, pc parser.Context) { + var hooks []astHook + + if b, ok := pc.Get(renderContextKey).(converter.RenderContext); ok && b.RenderTOC { + hooks = append(hooks, newTocAstHook(reader, pc)) + } + + if len(hooks) == 0 { + return + } + + ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + s := ast.WalkStatus(ast.WalkContinue) + + for i, hook := range hooks { + t, err := hook.Visit(n, entering) + if err != nil { + return t, err + } + if i == 0 || t > s { + s = t + } + } + + return s, nil + }) + + for _, hook := range hooks { + hook.Done() + } +} diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index 15b0f0d77c8..1586acbb34d 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -15,21 +15,22 @@ package goldmark import ( + "bufio" "bytes" "fmt" "path/filepath" "runtime/debug" + "github.com/gohugoio/hugo/identity" + "github.com/pkg/errors" "github.com/spf13/afero" "github.com/gohugoio/hugo/hugofs" - "github.com/alecthomas/chroma/styles" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/highlight" - "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/yuin/goldmark" hl "github.com/yuin/goldmark-highlighting" @@ -48,7 +49,7 @@ type provide struct { } func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) { - md := newMarkdown(cfg.MarkupConfig) + md := newMarkdown(cfg) return converter.NewProvider("goldmark", func(ctx converter.DocumentContext) (converter.Converter, error) { return &goldmarkConverter{ ctx: ctx, @@ -64,12 +65,14 @@ type goldmarkConverter struct { cfg converter.ProviderConfig } -func newMarkdown(mcfg markup_config.Config) goldmark.Markdown { - cfg := mcfg.Goldmark +func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown { + mcfg := pcfg.MarkupConfig + cfg := pcfg.MarkupConfig.Goldmark var ( extensions = []goldmark.Extender{ - newTocExtension(), + newLinks(), + newASTExtension(pcfg), } rendererOptions []renderer.Option parserOptions []parser.Option @@ -143,15 +146,53 @@ func newMarkdown(mcfg markup_config.Config) goldmark.Markdown { } +var _ identity.IdentitiesProvider = (*converterResult)(nil) + type converterResult struct { converter.Result toc tableofcontents.Root + ids identity.IdentitiesSet } func (c converterResult) TableOfContents() tableofcontents.Root { return c.toc } +func (c converterResult) GetIdentities() identity.IdentitiesSet { + return c.ids +} + +type renderContext struct { + util.BufWriter + renderContextData +} + +type renderContextData interface { + RenderContext() converter.RenderContext + DocumentContext() converter.DocumentContext + AddIdentity(id identity.Identity) +} + +type renderContextDataHolder struct { + rctx converter.RenderContext + dctx converter.DocumentContext + ids map[identity.Identity]bool +} + +func (ctx *renderContextDataHolder) RenderContext() converter.RenderContext { + return ctx.rctx +} + +func (ctx *renderContextDataHolder) DocumentContext() converter.DocumentContext { + return ctx.dctx +} + +func (ctx *renderContextDataHolder) AddIdentity(id identity.Identity) { + if _, found := ctx.ids[id]; !found { + ctx.ids[id] = true + } +} + func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result converter.Result, err error) { defer func() { if r := recover(); r != nil { @@ -166,9 +207,7 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert buf := &bytes.Buffer{} result = buf - pctx := parser.NewContext() - pctx.Set(tocEnableKey, ctx.RenderTOC) - + pctx := newParserContext(ctx) reader := text.NewReader(ctx.Src) doc := c.md.Parser().Parse( @@ -176,27 +215,58 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert parser.WithContext(pctx), ) - if err := c.md.Renderer().Render(buf, ctx.Src, doc); err != nil { + rcx := &renderContextDataHolder{ + rctx: ctx, + dctx: c.ctx, + ids: make(map[identity.Identity]bool), + } + + w := renderContext{ + BufWriter: bufio.NewWriter(buf), + renderContextData: rcx, + } + + if err := c.md.Renderer().Render(w, ctx.Src, doc); err != nil { return nil, err } - if toc, ok := pctx.Get(tocResultKey).(tableofcontents.Root); ok { - return converterResult{ - Result: buf, - toc: toc, - }, nil + return converterResult{ + Result: buf, + ids: rcx.ids, + toc: pctx.TableOfContents(), + }, nil + +} + +var featureSet = map[identity.Identity]bool{ + converter.FeatureRenderHooks: true, +} + +func (c *goldmarkConverter) Supports(feature identity.Identity) bool { + return featureSet[feature.GetIdentity()] +} + +func newParserContext(rctx converter.RenderContext) *parserContext { + ctx := parser.NewContext() + ctx.Set(renderContextKey, rctx) + return &parserContext{ + Context: ctx, } +} - return buf, nil +type parserContext struct { + parser.Context } -func newHighlighting(cfg highlight.Config) goldmark.Extender { - style := styles.Get(cfg.Style) - if style == nil { - style = styles.Fallback +func (p *parserContext) TableOfContents() tableofcontents.Root { + if v := p.Get(tocResultKey); v != nil { + return v.(tableofcontents.Root) } + return tableofcontents.Root{} +} - e := hl.NewHighlighting( +func newHighlighting(cfg highlight.Config) goldmark.Extender { + return hl.NewHighlighting( hl.WithStyle(cfg.Style), hl.WithGuessLanguage(cfg.GuessSyntax), hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()), @@ -230,6 +300,4 @@ func newHighlighting(cfg highlight.Config) goldmark.Extender { }), ) - - return e } diff --git a/markup/goldmark/convert_test.go b/markup/goldmark/convert_test.go index b6816d2e54a..62af8d809a2 100644 --- a/markup/goldmark/convert_test.go +++ b/markup/goldmark/convert_test.go @@ -38,6 +38,9 @@ func TestConvert(t *testing.T) { https://github.com/gohugoio/hugo/issues/6528 [Live Demo here!](https://docuapi.netlify.com/) +[I'm an inline-style link with title](https://www.google.com "Google's Homepage") + + ## Code Fences ยงยงยงbash @@ -98,6 +101,7 @@ description mconf := markup_config.Default mconf.Highlight.NoClasses = false + mconf.Goldmark.Renderer.Unsafe = true p, err := Provider.New( converter.ProviderConfig{ @@ -106,15 +110,15 @@ description }, ) c.Assert(err, qt.IsNil) - conv, err := p.New(converter.DocumentContext{}) + conv, err := p.New(converter.DocumentContext{DocumentID: "thedoc"}) c.Assert(err, qt.IsNil) - b, err := conv.Convert(converter.RenderContext{Src: []byte(content)}) + b, err := conv.Convert(converter.RenderContext{RenderTOC: true, Src: []byte(content)}) c.Assert(err, qt.IsNil) got := string(b.Bytes()) // Links - c.Assert(got, qt.Contains, `Live Demo here!`) + // c.Assert(got, qt.Contains, `Live Demo here!`) // Header IDs c.Assert(got, qt.Contains, `

Custom ID

`, qt.Commentf(got)) @@ -137,6 +141,11 @@ description c.Assert(got, qt.Contains, `
`) c.Assert(got, qt.Contains, `
date
`) + toc, ok := b.(converter.TableOfContentsProvider) + c.Assert(ok, qt.Equals, true) + tocHTML := toc.TableOfContents().ToHTML(1, 2) + c.Assert(tocHTML, qt.Contains, "TableOfContents") + } func TestCodeFence(t *testing.T) { diff --git a/markup/goldmark/render_link.go b/markup/goldmark/render_link.go new file mode 100644 index 00000000000..d78829050c5 --- /dev/null +++ b/markup/goldmark/render_link.go @@ -0,0 +1,204 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goldmark + +import ( + "github.com/gohugoio/hugo/markup/converter/hooks" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/util" +) + +var _ renderer.SetOptioner = (*linkRenderer)(nil) + +func newLinkRenderer() renderer.NodeRenderer { + r := &linkRenderer{ + Config: html.Config{ + Writer: html.DefaultWriter, + }, + } + return r +} + +func newLinks() goldmark.Extender { + return &links{} +} + +type linkContext struct { + page interface{} + destination string + title string + text string +} + +func (ctx linkContext) Destination() string { + return ctx.destination +} + +func (ctx linkContext) Page() interface{} { + return ctx.page +} + +func (ctx linkContext) Text() string { + return ctx.text +} + +func (ctx linkContext) Title() string { + return ctx.title +} + +type linkRenderer struct { + html.Config +} + +func (r *linkRenderer) SetOption(name renderer.OptionName, value interface{}) { + r.Config.SetOption(name, value) +} + +// RegisterFuncs implements NodeRenderer.RegisterFuncs. +func (r *linkRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindLink, r.renderLink) + reg.Register(ast.KindImage, r.renderImage) +} + +// Fall back to the default Goldmark render funcs. Method below borrowed from: +// https://github.com/yuin/goldmark/blob/b611cd333a492416b56aa8d94b04a67bf0096ab2/renderer/html/html.go#L404 +func (r *linkRenderer) renderDefaultImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.Image) + _, _ = w.WriteString("`) + _, _ = w.Write(n.Text(source)) + _ = w.WriteByte('") + } else { + _, _ = w.WriteString(">") + } + return ast.WalkSkipChildren, nil +} + +// Fall back to the default Goldmark render funcs. Method below borrowed from: +// https://github.com/yuin/goldmark/blob/b611cd333a492416b56aa8d94b04a67bf0096ab2/renderer/html/html.go#L404 +func (r *linkRenderer) renderDefaultLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Link) + if entering { + _, _ = w.WriteString("') + } else { + _, _ = w.WriteString("") + } + return ast.WalkContinue, nil +} + +func (r *linkRenderer) renderImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Image) + var h *hooks.Render + + ctx, ok := w.(renderContextData) + if ok { + h = ctx.RenderContext().RenderHooks + ok = h != nil && h.ImageRenderer != nil + } + + if !ok { + return r.renderDefaultImage(w, source, node, entering) + } + + if !entering { + return ast.WalkContinue, nil + } + + err := h.ImageRenderer.Render( + w, + linkContext{ + page: ctx.DocumentContext().Document, + destination: string(n.Destination), + title: string(n.Title), + text: string(n.Text(source)), + }, + ) + + ctx.AddIdentity(h.ImageRenderer.GetIdentity()) + + return ast.WalkSkipChildren, err + +} + +func (r *linkRenderer) renderLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Link) + var h *hooks.Render + + ctx, ok := w.(renderContextData) + if ok { + h = ctx.RenderContext().RenderHooks + ok = h != nil && h.LinkRenderer != nil + } + + if !ok { + return r.renderDefaultLink(w, source, node, entering) + } + + if !entering { + return ast.WalkContinue, nil + } + + err := h.LinkRenderer.Render( + w, + linkContext{ + page: ctx.DocumentContext().Document, + destination: string(n.Destination), + title: string(n.Title), + text: string(n.Text(source)), + }, + ) + + ctx.AddIdentity(h.LinkRenderer.GetIdentity()) + + // Do not render the inner text. + return ast.WalkSkipChildren, err + +} + +type links struct { +} + +// Extend implements goldmark.Extender. +func (e *links) Extend(m goldmark.Markdown) { + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(newLinkRenderer(), 100), + )) +} diff --git a/markup/goldmark/toc.go b/markup/goldmark/toc.go index 897f0098b6d..050b0636460 100644 --- a/markup/goldmark/toc.go +++ b/markup/goldmark/toc.go @@ -17,86 +17,79 @@ import ( "bytes" "github.com/gohugoio/hugo/markup/tableofcontents" - - "github.com/yuin/goldmark" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/text" - "github.com/yuin/goldmark/util" ) var ( - tocResultKey = parser.NewContextKey() - tocEnableKey = parser.NewContextKey() + tocResultKey = parser.NewContextKey() + renderContextKey = parser.NewContextKey() ) -type tocTransformer struct { +// TODO1 revert the content spec changes + +type tocAstHook struct { + reader text.Reader + pc parser.Context + + // ToC state + toc tableofcontents.Root + header tableofcontents.Header + level int + row int + inHeading bool + headingText bytes.Buffer } -func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parser.Context) { - if b, ok := pc.Get(tocEnableKey).(bool); !ok || !b { - return +func newTocAstHook(reader text.Reader, pc parser.Context) *tocAstHook { + return &tocAstHook{ + reader: reader, + pc: pc, + row: -1, } +} - var ( - toc tableofcontents.Root - header tableofcontents.Header - level int - row = -1 - inHeading bool - headingText bytes.Buffer - ) - - ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) { - s := ast.WalkStatus(ast.WalkContinue) - if n.Kind() == ast.KindHeading { - if inHeading && !entering { - header.Text = headingText.String() - headingText.Reset() - toc.AddAt(header, row, level-1) - header = tableofcontents.Header{} - inHeading = false - return s, nil - } - - inHeading = true - } - - if !(inHeading && entering) { +func (h *tocAstHook) Visit(n ast.Node, entering bool) (ast.WalkStatus, error) { + s := ast.WalkStatus(ast.WalkContinue) + if n.Kind() == ast.KindHeading { + if h.inHeading && !entering { + h.header.Text = h.headingText.String() + h.headingText.Reset() + h.toc.AddAt(h.header, h.row, h.level-1) + h.header = tableofcontents.Header{} + h.inHeading = false return s, nil } - switch n.Kind() { - case ast.KindHeading: - heading := n.(*ast.Heading) - level = heading.Level - - if level == 1 || row == -1 { - row++ - } - - id, found := heading.AttributeString("id") - if found { - header.ID = string(id.([]byte)) - } - case ast.KindText: - textNode := n.(*ast.Text) - headingText.Write(textNode.Text(reader.Source())) - } + h.inHeading = true + } + if !(h.inHeading && entering) { return s, nil - }) + } - pc.Set(tocResultKey, toc) -} + switch n.Kind() { + case ast.KindHeading: + heading := n.(*ast.Heading) + h.level = heading.Level -type tocExtension struct { -} + if h.level == 1 || h.row == -1 { + h.row++ + } + + id, found := heading.AttributeString("id") + if found { + h.header.ID = string(id.([]byte)) + } + case ast.KindText: + textNode := n.(*ast.Text) + h.headingText.Write(textNode.Text(h.reader.Source())) + } -func newTocExtension() goldmark.Extender { - return &tocExtension{} + return s, nil } -func (e *tocExtension) Extend(m goldmark.Markdown) { - m.Parser().AddOptions(parser.WithASTTransformers(util.Prioritized(&tocTransformer{}, 10))) +func (h *tocAstHook) Done() { + h.pc.Set(tocResultKey, h.toc) } diff --git a/markup/mmark/convert.go b/markup/mmark/convert.go index 07b2a6f81e5..0682ad276c6 100644 --- a/markup/mmark/convert.go +++ b/markup/mmark/convert.go @@ -15,6 +15,7 @@ package mmark import ( + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config" "github.com/gohugoio/hugo/markup/converter" "github.com/miekg/mmark" @@ -65,6 +66,10 @@ func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, return mmark.Parse(ctx.Src, r, c.extensions), nil } +func (c *mmarkConverter) Supports(feature identity.Identity) bool { + return false +} + func getHTMLRenderer( ctx converter.DocumentContext, cfg blackfriday_config.Config, diff --git a/markup/org/convert.go b/markup/org/convert.go index 4d6e5e2fa0f..2b1fbb73c3a 100644 --- a/markup/org/convert.go +++ b/markup/org/convert.go @@ -17,6 +17,8 @@ package org import ( "bytes" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter" "github.com/niklasfasching/go-org/org" "github.com/spf13/afero" @@ -66,3 +68,7 @@ func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, e } return converter.Bytes([]byte(html)), nil } + +func (c *orgConverter) Supports(feature identity.Identity) bool { + return false +} diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index d538d4a5265..d6d5ab18c8c 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -17,6 +17,7 @@ package pandoc import ( "os/exec" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -47,6 +48,10 @@ func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil } +func (c *pandocConverter) Supports(feature identity.Identity) bool { + return false +} + // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte { logger := c.cfg.Logger diff --git a/markup/rst/convert.go b/markup/rst/convert.go index 040b40d792d..64cc8b5114f 100644 --- a/markup/rst/convert.go +++ b/markup/rst/convert.go @@ -19,6 +19,7 @@ import ( "os/exec" "runtime" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" "github.com/gohugoio/hugo/markup/converter" @@ -48,6 +49,10 @@ func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, e return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil } +func (c *rstConverter) Supports(feature identity.Identity) bool { + return false +} + // getRstContent calls the Python script rst2html as an external helper // to convert reStructuredText content to HTML. func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte { diff --git a/output/layout.go b/output/layout.go index 055d742b15f..7d935d0ba89 100644 --- a/output/layout.go +++ b/output/layout.go @@ -37,6 +37,12 @@ type LayoutDescriptor struct { Layout string // LayoutOverride indicates what we should only look for the above layout. LayoutOverride bool + + RenderingHook bool +} + +func (d LayoutDescriptor) isList() bool { + return !d.RenderingHook && d.Kind != "page" } // LayoutHandler calculates the layout template to use to render a given output type. @@ -89,7 +95,7 @@ type layoutBuilder struct { func (l *layoutBuilder) addLayoutVariations(vars ...string) { for _, layoutVar := range vars { - if l.d.LayoutOverride && layoutVar != l.d.Layout { + if !l.d.RenderingHook && l.d.LayoutOverride && layoutVar != l.d.Layout { continue } l.layoutVariations = append(l.layoutVariations, layoutVar) @@ -99,6 +105,9 @@ func (l *layoutBuilder) addLayoutVariations(vars ...string) { func (l *layoutBuilder) addTypeVariations(vars ...string) { for _, typeVar := range vars { if !reservedSections[typeVar] { + if l.d.RenderingHook { + typeVar = typeVar + renderingHookRoot + } l.typeVariations = append(l.typeVariations, typeVar) } } @@ -115,16 +124,25 @@ func (l *layoutBuilder) addKind() { l.addTypeVariations(l.d.Kind) } +const renderingHookRoot = "/_markup" + func resolvePageTemplate(d LayoutDescriptor, f Format) []string { b := &layoutBuilder{d: d, f: f} - if d.Layout != "" { - b.addLayoutVariations(d.Layout) - } - - if d.Type != "" { - b.addTypeVariations(d.Type) + if d.RenderingHook { + if d.Type != "" { + b.addTypeVariations(d.Type) + } + b.addLayoutVariations(d.Kind) + b.addSectionType() + } else { + if d.Layout != "" { + b.addLayoutVariations(d.Layout) + } + if d.Type != "" { + b.addTypeVariations(d.Type) + } } switch d.Kind { @@ -159,7 +177,7 @@ func resolvePageTemplate(d LayoutDescriptor, f Format) []string { } isRSS := f.Name == RSSFormat.Name - if isRSS { + if !d.RenderingHook && isRSS { // The historic and common rss.xml case b.addLayoutVariations("") } @@ -167,14 +185,14 @@ func resolvePageTemplate(d LayoutDescriptor, f Format) []string { // All have _default in their lookup path b.addTypeVariations("_default") - if d.Kind != "page" { + if d.isList() { // Add the common list type b.addLayoutVariations("list") } layouts := b.resolveVariations() - if isRSS { + if !d.RenderingHook && isRSS { layouts = append(layouts, "_internal/_default/rss.xml") } diff --git a/output/layout_test.go b/output/layout_test.go index c6267b27434..9e4f89098c3 100644 --- a/output/layout_test.go +++ b/output/layout_test.go @@ -111,6 +111,8 @@ func TestLayout(t *testing.T) { []string{"section/shortcodes.amp.html"}, 12}, {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, "", ampType, []string{"section/partials.amp.html"}, 12}, + {"Content hook", LayoutDescriptor{Kind: "render-link", RenderingHook: true, Section: "blog"}, "", ampType, + []string{"blog/_markup/render-link.amp.html", "blog/_markup/render-link.html", "_default/_markup/render-link.amp.html", "_default/_markup/render-link.html"}, 4}, } { c.Run(this.name, func(c *qt.C) { l := NewLayoutHandler() diff --git a/public/categories/index.xml b/public/categories/index.xml new file mode 100644 index 00000000000..ae8c7f8f749 --- /dev/null +++ b/public/categories/index.xml @@ -0,0 +1,13 @@ + + + + Categories on + /categories/ + Recent content in Categories on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/public/index.xml b/public/index.xml new file mode 100644 index 00000000000..b70aeed6e08 --- /dev/null +++ b/public/index.xml @@ -0,0 +1,13 @@ + + + + + / + Recent content on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/public/sitemap.xml b/public/sitemap.xml new file mode 100644 index 00000000000..95cee9f7cdc --- /dev/null +++ b/public/sitemap.xml @@ -0,0 +1,17 @@ + + + + + / + + + + /categories/ + + + + /tags/ + + + \ No newline at end of file diff --git a/public/tags/index.xml b/public/tags/index.xml new file mode 100644 index 00000000000..43dbc43baa7 --- /dev/null +++ b/public/tags/index.xml @@ -0,0 +1,13 @@ + + + + Tags on + /tags/ + Recent content in Tags on + Hugo -- gohugo.io + + + + + + \ No newline at end of file diff --git a/tpl/template_info.go b/tpl/template_info.go index be056695895..a7bb7f5476b 100644 --- a/tpl/template_info.go +++ b/tpl/template_info.go @@ -13,6 +13,10 @@ package tpl +import ( + "github.com/gohugoio/hugo/identity" +) + // Increments on breaking changes. const TemplateVersion = 2 @@ -27,6 +31,8 @@ type Info struct { // Config extracted from template. Config Config + + identity.Manager } func (info Info) IsZero() bool { diff --git a/tpl/tplimpl/ace.go b/tpl/tplimpl/ace.go index bdbc7105992..63c36b7cb66 100644 --- a/tpl/tplimpl/ace.go +++ b/tpl/tplimpl/ace.go @@ -53,15 +53,14 @@ func (t *templateHandler) addAceTemplate(name, basePath, innerPath string, baseC typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToHMLTTemplate(typ, templ) + c, err := t.applyTemplateTransformersToHMLTTemplate(typ, templ) if err != nil { return err } + t.templateInfo[name] = c.Info if typ == templateShortcode { t.addShortcodeVariant(name, c.Info, templ) - } else { - t.templateInfo[name] = c.Info } return nil diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 6027775243d..69ca2d800b4 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -18,7 +18,10 @@ import ( "html/template" "strings" texttemplate "text/template" - "text/template/parse" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/tpl/tplimpl/embedded" @@ -274,8 +277,8 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler { templateInfo: t.templateInfo, html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template), templatesCommon: t.html.templatesCommon}, text: &textTemplates{ - textTemplate: &textTemplate{t: texttemplate.Must(t.text.t.Clone())}, - standalone: &textTemplate{t: texttemplate.New("")}, + textTemplate: &textTemplate{templates: t.text, t: texttemplate.Must(t.text.t.Clone())}, + standalone: &textTemplate{templates: t.text, t: texttemplate.New("")}, overlays: make(map[string]*texttemplate.Template), templatesCommon: t.text.templatesCommon}, errors: make([]*templateErr, 0), } @@ -324,6 +327,7 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { common := &templatesCommon{ nameBaseTemplateName: make(map[string]string), transformNotFound: make(map[string]bool), + identityNotFound: make(map[string][]tpl.Info), } htmlT := &htmlTemplates{ @@ -347,6 +351,8 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { errors: make([]*templateErr, 0), } + textT.textTemplate.templates = textT + textT.standalone.templates = textT common.handler = h return h @@ -364,7 +370,10 @@ type templatesCommon struct { // Holds names of the templates not found during the first AST transformation // pass. transformNotFound map[string]bool + + identityNotFound map[string][]tpl.Info } + type htmlTemplates struct { mu sync.RWMutex @@ -504,15 +513,19 @@ func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) ( typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToHMLTTemplate(typ, templ) + c, err := t.handler.applyTemplateTransformersToHMLTTemplate(typ, templ) if err != nil { return nil, err } - for k := range c.notFound { + for k := range c.templateNotFound { t.transformNotFound[k] = true } + for k := range c.identityNotFound { + t.identityNotFound[k] = append(t.identityNotFound[k], c.Info) + } + if typ == templateShortcode { t.handler.addShortcodeVariant(name, c.Info, templ) } else { @@ -532,8 +545,9 @@ func (t *htmlTemplates) addLateTemplate(name, tpl string) error { } type textTemplate struct { - mu sync.RWMutex - t *texttemplate.Template + mu sync.RWMutex + t *texttemplate.Template + templates *textTemplates } func (t *textTemplate) Parse(name, tpl string) (tpl.Template, error) { @@ -557,7 +571,7 @@ func (t *textTemplate) parseIn(tt *texttemplate.Template, name, tpl string) (*te return nil, err } - if _, err := applyTemplateTransformersToTextTemplate(templateUndefined, templ); err != nil { + if _, err := t.templates.handler.applyTemplateTransformersToTextTemplate(templateUndefined, templ); err != nil { return nil, err } return templ, nil @@ -572,12 +586,12 @@ func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl strin typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToTextTemplate(typ, templ) + c, err := t.handler.applyTemplateTransformersToTextTemplate(typ, templ) if err != nil { return nil, err } - for k := range c.notFound { + for k := range c.templateNotFound { t.transformNotFound[k] = true } @@ -603,36 +617,60 @@ func (t *templateHandler) addTemplate(name, tpl string) error { return t.AddTemplate(name, tpl) } -func (t *templateHandler) postTransform() error { - if len(t.html.transformNotFound) == 0 && len(t.text.transformNotFound) == 0 { - return nil +func (t *templateHandler) getOrCreateTemplateInfo(name string) tpl.Info { + info, found := t.templateInfo[name] + if found { + return info } + info = newTemplateInfo(name) + t.templateInfo[name] = info + return info +} +func newTemplateInfo(name string) tpl.Info { + return tpl.Info{ + Manager: identity.NewIdentityManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name)), + Config: tpl.DefaultConfig, + } +} + +func (t *templateHandler) postTransform() error { defer func() { t.text.transformNotFound = make(map[string]bool) t.html.transformNotFound = make(map[string]bool) + t.text.identityNotFound = make(map[string][]tpl.Info) + t.html.identityNotFound = make(map[string][]tpl.Info) }() for _, s := range []struct { - lookup func(name string) *parse.Tree + lookup func(name string) *templateInfoTree transformNotFound map[string]bool + identityNotFound map[string][]tpl.Info }{ // html templates - {func(name string) *parse.Tree { + {func(name string) *templateInfoTree { templ := t.html.lookup(name) if templ == nil { return nil } - return templ.Tree - }, t.html.transformNotFound}, + info := t.getOrCreateTemplateInfo(name) + return &templateInfoTree{ + info: info, + tree: templ.Tree, + } + }, t.html.transformNotFound, t.html.identityNotFound}, // text templates - {func(name string) *parse.Tree { + {func(name string) *templateInfoTree { templT := t.text.lookup(name) if templT == nil { return nil } - return templT.Tree - }, t.text.transformNotFound}, + info := t.getOrCreateTemplateInfo(name) + return &templateInfoTree{ + info: info, + tree: templT.Tree, + } + }, t.text.transformNotFound, t.text.identityNotFound}, } { for name := range s.transformNotFound { templ := s.lookup(name) @@ -643,6 +681,16 @@ func (t *templateHandler) postTransform() error { } } } + + for k, v := range s.identityNotFound { + templ := s.lookup(k) + if templ != nil { + id := templ.info.GetIdentity() + for _, im := range v { + im.Add(id) + } + } + } } return nil @@ -677,8 +725,6 @@ func (t *templateHandler) AddTemplate(name, tpl string) error { // MarkReady marks the templates as "ready for execution". No changes allowed // after this is set. -// TODO(bep) if this proves to be resource heavy, we could detect -// earlier if we really need this, or make it lazy. func (t *templateHandler) MarkReady() error { if err := t.postTransform(); err != nil { return err @@ -839,7 +885,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin // * https://github.com/golang/go/issues/16101 // * https://github.com/gohugoio/hugo/issues/2549 overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - if _, err := applyTemplateTransformersToHMLTTemplate(templateUndefined, overlayTpl); err != nil { + if _, err := t.handler.applyTemplateTransformersToHMLTTemplate(templateUndefined, overlayTpl); err != nil { return err } @@ -879,7 +925,7 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin } overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - if _, err := applyTemplateTransformersToTextTemplate(templateUndefined, overlayTpl); err != nil { + if _, err := t.handler.applyTemplateTransformersToTextTemplate(templateUndefined, overlayTpl); err != nil { return err } t.overlays[name] = overlayTpl @@ -951,7 +997,7 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e typ := resolveTemplateType(name) - c, err := applyTemplateTransformersToHMLTTemplate(typ, templ) + c, err := t.applyTemplateTransformersToHMLTTemplate(typ, templ) if err != nil { return err } diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index e25e70e350e..5e11db78b05 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -47,10 +47,11 @@ const ( ) type templateContext struct { - decl decl - visited map[string]bool - notFound map[string]bool - lookupFn func(name string) *parse.Tree + decl decl + visited map[string]bool + templateNotFound map[string]bool + identityNotFound map[string]bool + lookupFn func(name string) *templateInfoTree // The last error encountered. err error @@ -67,7 +68,7 @@ type templateContext struct { returnNode *parse.CommandNode } -func (c templateContext) getIfNotVisited(name string) *parse.Tree { +func (c templateContext) getIfNotVisited(name string) *templateInfoTree { if c.visited[name] { return nil } @@ -77,60 +78,98 @@ func (c templateContext) getIfNotVisited(name string) *parse.Tree { // This may be a inline template defined outside of this file // and not yet parsed. Unusual, but it happens. // Store the name to try again later. - c.notFound[name] = true + c.templateNotFound[name] = true } return templ } -func newTemplateContext(lookupFn func(name string) *parse.Tree) *templateContext { +func newTemplateContext(info tpl.Info, lookupFn func(name string) *templateInfoTree) *templateContext { + if info.Manager == nil { + panic("identity manager not set") + } return &templateContext{ - Info: tpl.Info{Config: tpl.DefaultConfig}, - lookupFn: lookupFn, - decl: make(map[string]string), - visited: make(map[string]bool), - notFound: make(map[string]bool)} + Info: info, + lookupFn: lookupFn, + decl: make(map[string]string), + visited: make(map[string]bool), + templateNotFound: make(map[string]bool), + identityNotFound: make(map[string]bool), + } +} + +func createParseTreeLookup(templ *template.Template) func(nn string) *templateInfoTree { + return createParseTreeLookupFor(templ, func(name string) tpl.Info { return newTemplateInfo(name) }) + } -func createParseTreeLookup(templ *template.Template) func(nn string) *parse.Tree { - return func(nn string) *parse.Tree { +func createParseTreeLookupFor(templ *template.Template, infoFn func(name string) tpl.Info) func(nn string) *templateInfoTree { + return func(nn string) *templateInfoTree { tt := templ.Lookup(nn) if tt != nil { - return tt.Tree + return &templateInfoTree{ + tree: tt.Tree, + info: infoFn(nn), + } } return nil } } -func applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) { - return applyTemplateTransformers(typ, templ.Tree, createParseTreeLookup(templ)) +func (t *templateHandler) createParseTreeLookup(templ *template.Template) func(nn string) *templateInfoTree { + return createParseTreeLookupFor(templ, func(name string) tpl.Info { return t.templateInfo[name] }) } -func applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) { - return applyTemplateTransformers(typ, templ.Tree, - func(nn string) *parse.Tree { +func (t *templateHandler) applyTemplateTransformersToHMLTTemplate(typ templateType, templ *template.Template) (*templateContext, error) { + ti := &templateInfoTree{ + tree: templ.Tree, + info: t.getOrCreateTemplateInfo(templ.Name()), + } + return applyTemplateTransformers(typ, ti, t.createParseTreeLookup(templ)) +} + +func (t *templateHandler) applyTemplateTransformersToTextTemplate(typ templateType, templ *texttemplate.Template) (*templateContext, error) { + ti := &templateInfoTree{ + tree: templ.Tree, + info: t.getOrCreateTemplateInfo(templ.Name()), + } + + return applyTemplateTransformers(typ, ti, + func(nn string) *templateInfoTree { tt := templ.Lookup(nn) if tt != nil { - return tt.Tree + return &templateInfoTree{ + tree: tt.Tree, + info: t.getOrCreateTemplateInfo(nn), + } } return nil }) } -func applyTemplateTransformers(typ templateType, templ *parse.Tree, lookupFn func(name string) *parse.Tree) (*templateContext, error) { +type templateInfoTree struct { + info tpl.Info + tree *parse.Tree +} + +func applyTemplateTransformers( + typ templateType, + templ *templateInfoTree, + lookupFn func(name string) *templateInfoTree) (*templateContext, error) { + if templ == nil { return nil, errors.New("expected template, but none provided") } - c := newTemplateContext(lookupFn) + c := newTemplateContext(templ.info, lookupFn) c.typ = typ - _, err := c.applyTransformations(templ.Root) + _, err := c.applyTransformations(templ.tree.Root) if err == nil && c.returnNode != nil { // This is a partial with a return statement. c.Info.HasReturn = true - templ.Root = c.wrapInPartialReturnWrapper(templ.Root) + templ.tree.Root = c.wrapInPartialReturnWrapper(templ.tree.Root) } return c, err @@ -213,7 +252,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { case *parse.TemplateNode: subTempl := c.getIfNotVisited(x.Name) if subTempl != nil { - c.applyTransformationsToNodes(subTempl.Root) + c.applyTransformationsToNodes(subTempl.tree.Root) } case *parse.PipeNode: c.collectConfig(x) @@ -230,6 +269,26 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: + if len(x.Args) > 1 { + if id, ok := x.Args[0].(*parse.IdentifierNode); ok { + if id.Ident == "partial" { + partialName := strings.Trim(x.Args[1].String(), "\"") + if !strings.Contains(partialName, ".") { + partialName += ".html" + } + // TODO1 add a test for case + partialName = "partials/" + partialName + info := c.lookupFn(partialName) + if info != nil { + c.Info.Add(info.info) + } else { + // Delay for later + c.identityNotFound[partialName] = true + } + } + } + } + c.collectInner(x) keep := c.collectReturnNode(x) diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go index 682af277239..76f6f54977a 100644 --- a/tpl/tplimpl/template_ast_transformers_test.go +++ b/tpl/tplimpl/template_ast_transformers_test.go @@ -21,11 +21,19 @@ import ( "github.com/gohugoio/hugo/tpl" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/cast" qt "github.com/frankban/quicktest" ) +var eq = qt.CmpEquals( + cmp.Comparer(func(i1, i2 tpl.Info) bool { + return cmp.Equal(i1, i2, cmpopts.IgnoreFields(tpl.Info{}, "Manager")) + }), +) + type paramsHolder struct { params map[string]interface{} page *paramsHolder @@ -218,7 +226,7 @@ func TestParamsKeysToLower(t *testing.T) { c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templ)) c.Assert(ctx.decl.indexOfReplacementStart([]string{}), qt.Equals, -1) @@ -307,7 +315,7 @@ func BenchmarkTemplateParamsKeysToLower(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - c := newTemplateContext(createParseTreeLookup(templates[i])) + c := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templates[i])) c.applyTransformations(templ.Tree.Root) } } @@ -347,7 +355,7 @@ Pretty First3: {{ $__amber_4.COLORS.PRETTY.FIRST}} c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templ)) ctx.applyTransformations(templ.Tree.Root) @@ -392,7 +400,7 @@ P2: {{ .Params.LOWER }} c.Assert(err, qt.IsNil) overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - ctx := newTemplateContext(createParseTreeLookup(overlayTpl)) + ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(overlayTpl)) ctx.applyTransformations(overlayTpl.Tree.Root) @@ -414,7 +422,7 @@ func TestTransformRecursiveTemplate(t *testing.T) { {{ define "menu-nodes" }} {{ template "menu-node" }} {{ end }} -{{ define "menu-node" }} +{{ define "menu-nรŸode" }} {{ template "menu-node" }} {{ end }} {{ template "menu-nodes" }} @@ -423,7 +431,10 @@ func TestTransformRecursiveTemplate(t *testing.T) { templ, err := template.New("foo").Parse(recursive) c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext( + newTemplateInfo("test"), + createParseTreeLookup(templ), + ) ctx.applyTransformations(templ.Tree.Root) } @@ -540,11 +551,12 @@ func TestCollectInfo(t *testing.T) { templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString) c.Assert(err, qt.IsNil) - ctx := newTemplateContext(createParseTreeLookup(templ)) + ctx := newTemplateContext( + newTemplateInfo("test"), createParseTreeLookup(templ)) ctx.typ = templateShortcode ctx.applyTransformations(templ.Tree.Root) - c.Assert(ctx.Info, qt.Equals, test.expected) + c.Assert(ctx.Info, eq, test.expected) }) } @@ -582,7 +594,7 @@ func TestPartialReturn(t *testing.T) { templ, err := template.New("foo").Funcs(funcs).Parse(test.tplString) c.Assert(err, qt.IsNil) - _, err = applyTemplateTransformers(templatePartial, templ.Tree, createParseTreeLookup(templ)) + _, err = applyTemplateTransformers(templatePartial, &templateInfoTree{tree: templ.Tree, info: newTemplateInfo("test")}, createParseTreeLookup(templ)) // Just check that it doesn't fail in this test. We have functional tests // in hugoblib. diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 10fbc2375c0..2e08bbbe289 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -101,6 +101,7 @@ func TestTemplateFuncsExamples(t *testing.T) { depsCfg.Fs = fs d, err := deps.New(depsCfg) c.Assert(err, qt.IsNil) + c.Assert(err, qt.IsNil) var data struct { Title string From 11d544871d6f0ecc05a5370166568f678616a308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 5 Dec 2019 16:53:24 +0100 Subject: [PATCH 02/19] RenderString --- hugolib/content_render_hooks_test.go | 15 ++++++++++++++ hugolib/page.go | 31 +++++++++++++++++++++------- hugolib/page__per_output.go | 13 ++++-------- hugolib/shortcode.go | 9 +------- resources/page/page.go | 5 +++-- resources/page/page_nop.go | 8 +++++-- resources/page/testhelpers_test.go | 6 +++++- 7 files changed, 58 insertions(+), 29 deletions(-) diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index 74b054d757a..5cb79f0506c 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -31,6 +31,12 @@ workingDir="/mywork" {{ .Inner | markdownify }} `) + b.WithTemplatesAdded("shortcodes/myshortcode5.html", ` +
+{{ .Inner | .Page.RenderString }} +
+`) + b.WithTemplatesAdded("partials/mypartial1.html", `PARTIAL1`) b.WithTemplatesAdded("partials/mypartial2.html", `PARTIAL2 {{ partial "mypartial3.html" }}`) b.WithTemplatesAdded("partials/mypartial3.html", `PARTIAL3`) @@ -96,6 +102,12 @@ title: Cool Page With Markdownify Inner Link: [Inner Link](https://www.google.com "Google's Homepage") {{< /myshortcode4 >}} +`, "blog/p6.md", `--- +title: With RenderString +--- + +{{< myshortcode5 >}}Inner Link: [Inner Link](https://www.google.com "Google's Homepage"){{< /myshortcode5 >}} + `) b.Build(BuildCfg{}) b.AssertFileContent("public/blog/p1/index.html", ` @@ -111,6 +123,8 @@ SHORT3| // The regular markdownify func currently gets regular links. b.AssertFileContent("public/blog/p5/index.html", "Inner Link: Inner Link\n") + b.AssertFileContent("public/blog/p6/index.html", "
\n

Inner Link: With RenderString|https://www.google.com|Title: Google's Homepage|Text: Inner Link|END

\n\n
") + b.EditFiles( "layouts/_default/_markup/render-link.html", `EDITED: {{ .Destination | safeURL }}|`, "layouts/_default/_markup/render-image.html", `IMAGE EDITED: {{ .Destination | safeURL }}|`, @@ -126,5 +140,6 @@ SHORT3| b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3_EDITED`) b.AssertFileContent("public/docs/docs1/index.html", `DOCS EDITED: https://www.google.com|

`) b.AssertFileContent("public/blog/p4/index.html", `IMAGE EDITED: /images/Dragster.jpg|`) + b.AssertFileContent("public/blog/p6/index.html", "FOO") } diff --git a/hugolib/page.go b/hugolib/page.go index 625206ad114..ce28988fff0 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -23,6 +23,8 @@ import ( "sort" "strings" + "github.com/spf13/cast" + "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/identity" @@ -565,11 +567,27 @@ func (p *pageState) AlternativeOutputFormats() page.OutputFormats { return o } -func (p *pageState) Render(layout ...string) template.HTML { +func (p *pageState) RenderString(in interface{}) (template.HTML, error) { + s, err := cast.ToStringE(in) + if err != nil { + return "", p.wrapError(err) + } + + b, err := p.pageOutput.cp.renderContent([]byte(s), false) + if err != nil { + return "", p.wrapError(err) + } + + if str, ok := b.(fmt.Stringer); ok { + return template.HTML(str.String()), nil + } + return template.HTML(b.Bytes()), nil +} + +func (p *pageState) Render(layout ...string) (template.HTML, error) { l, err := p.getLayouts(layout...) if err != nil { - p.s.SendError(p.wrapError(errors.Errorf(".Render: failed to resolve layout %v", layout))) - return "" + return "", p.wrapError(errors.Errorf("failed to resolve layout %v", layout)) } for _, layout := range l { @@ -583,14 +601,13 @@ func (p *pageState) Render(layout ...string) template.HTML { if templ != nil { res, err := executeToString(templ, p) if err != nil { - p.s.SendError(p.wrapError(errors.Wrapf(err, ".Render: failed to execute template %q v", layout))) - return "" + return "", p.wrapError(errors.Wrapf(err, "failed to execute template %q v", layout)) } - return template.HTML(res) + return template.HTML(res), nil } } - return "" + return "", nil } diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index a9639c04747..91d5db8d9c8 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -110,7 +110,7 @@ func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput if p.renderable { if !isHTML { - r, err := cp.renderContent(cp.workContent) + r, err := cp.renderContent(cp.workContent, true) if err != nil { return err } @@ -161,12 +161,7 @@ func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput } } } else if cp.p.m.summary != "" { - b, err := cp.p.getContentConverter().Convert( - converter.RenderContext{ - Src: []byte(cp.p.m.summary), - }, - ) - + b, err := cp.renderContent([]byte(cp.p.m.summary), false) if err != nil { return err } @@ -354,12 +349,12 @@ func (p *pageContentOutput) setAutoSummary() error { } -func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) { +func (cp *pageContentOutput) renderContent(content []byte, renderTOC bool) (converter.Result, error) { c := cp.p.getContentConverter() return c.Convert( converter.RenderContext{ Src: content, - RenderTOC: true, + RenderTOC: renderTOC, RenderHooks: cp.renderHooks, }) } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 5e916aeec24..338c30b698f 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -23,8 +23,6 @@ import ( "html/template" "path" - "github.com/gohugoio/hugo/markup/converter" - "github.com/gohugoio/hugo/common/herrors" "github.com/pkg/errors" @@ -351,12 +349,7 @@ func renderShortcode( // shortcode. if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) { var err error - - b, err := p.getContentConverter().Convert( - converter.RenderContext{ - Src: []byte(inner), - }, - ) + b, err := p.pageOutput.cp.renderContent([]byte(inner), false) if err != nil { return "", false, err diff --git a/resources/page/page.go b/resources/page/page.go index 3b43b0af3f1..2ae23792e29 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -201,9 +201,10 @@ type PageMetaProvider interface { Weight() int } -// PageRenderProvider provides a way for a Page to render itself. +// PageRenderProvider provides a way for a Page to render content. type PageRenderProvider interface { - Render(layout ...string) template.HTML + Render(layout ...string) (template.HTML, error) + RenderString(s interface{}) (template.HTML, error) // TODO1 inline option? } // PageWithoutContent is the Page without any of the content methods. diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go index 09ac136fc2b..43a7b7b4689 100644 --- a/resources/page/page_nop.go +++ b/resources/page/page_nop.go @@ -371,8 +371,12 @@ func (p *nopPage) RelRef(argsm map[string]interface{}) (string, error) { return "", nil } -func (p *nopPage) Render(layout ...string) template.HTML { - return "" +func (p *nopPage) Render(layout ...string) (template.HTML, error) { + return "", nil +} + +func (p *nopPage) RenderString(s interface{}) (template.HTML, error) { + return "", nil } func (p *nopPage) ResourceType() string { diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go index cc6a74f06de..746a881f9d2 100644 --- a/resources/page/testhelpers_test.go +++ b/resources/page/testhelpers_test.go @@ -446,7 +446,11 @@ func (p *testPage) RelRefFrom(argsm map[string]interface{}, source interface{}) return "", nil } -func (p *testPage) Render(layout ...string) template.HTML { +func (p *testPage) Render(layout ...string) (template.HTML, error) { + panic("not implemented") +} + +func (p *testPage) RenderString(s interface{}) (template.HTML, error) { panic("not implemented") } From 7c9358e98a670e70fe3d7a51004f328aa1c02c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 5 Dec 2019 18:20:15 +0100 Subject: [PATCH 03/19] Work --- hugolib/content_render_hooks_test.go | 7 +- hugolib/hugo_sites.go | 37 +++------- hugolib/page.go | 1 + hugolib/page__per_output.go | 44 ++++++++++-- hugolib/site.go | 4 +- identity/identity.go | 103 ++++++++++++++------------- magefile.go | 2 +- markup/converter/converter.go | 4 +- markup/converter/hooks/hooks.go | 10 +-- markup/goldmark/convert.go | 16 ++--- markup/goldmark/render_link.go | 4 ++ tpl/tplimpl/template.go | 13 +++- 12 files changed, 136 insertions(+), 109 deletions(-) diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index 5cb79f0506c..bcb18576e54 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -106,7 +106,7 @@ Inner Link: [Inner Link](https://www.google.com "Google's Homepage") title: With RenderString --- -{{< myshortcode5 >}}Inner Link: [Inner Link](https://www.google.com "Google's Homepage"){{< /myshortcode5 >}} +{{< myshortcode5 >}}Inner Link: [Inner Link](https://www.gohugo.io "Hugo's Homepage"){{< /myshortcode5 >}} `) b.Build(BuildCfg{}) @@ -123,7 +123,7 @@ SHORT3| // The regular markdownify func currently gets regular links. b.AssertFileContent("public/blog/p5/index.html", "Inner Link: Inner Link\n") - b.AssertFileContent("public/blog/p6/index.html", "
\n

Inner Link: With RenderString|https://www.google.com|Title: Google's Homepage|Text: Inner Link|END

\n\n
") + b.AssertFileContent("public/blog/p6/index.html", "
\n

Inner Link: With RenderString|https://www.gohugo.io|Title: Hugo's Homepage|Text: Inner Link|END

\n\n
") b.EditFiles( "layouts/_default/_markup/render-link.html", `EDITED: {{ .Destination | safeURL }}|`, @@ -134,12 +134,11 @@ SHORT3| "layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`, ) b.Build(BuildCfg{}) - b.AssertFileContent("public/blog/p1/index.html", `

EDITED: https://www.google.com|

`, "SHORT3_EDITED|") b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`) b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3_EDITED`) b.AssertFileContent("public/docs/docs1/index.html", `DOCS EDITED: https://www.google.com|

`) b.AssertFileContent("public/blog/p4/index.html", `IMAGE EDITED: /images/Dragster.jpg|`) - b.AssertFileContent("public/blog/p6/index.html", "FOO") + b.AssertFileContent("public/blog/p6/index.html", "

Inner Link: EDITED: https://www.gohugo.io|

") } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 479ff4bcd2f..2368d7e4f6c 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -807,43 +807,28 @@ func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Page return h.Sites[0].findPagesByKindIn(kind, inPages) } -func (h *HugoSites) resetPageStateFromEvents(ids identity.Identities) { - idset := ids.ToIdentitySet() - hasIdentify := func(v interface{}) bool { - if id, ok := v.(identity.Provider); ok { - if idset[id.GetIdentity()] { - return true - } - } - if idp, ok := v.(identity.IdentitiesProvider); ok { - for id, _ := range idp.GetIdentities() { - if idset[id.GetIdentity()] { - return true - } - } - } - return false - } +func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { for _, s := range h.Sites { PAGES: for _, p := range s.rawAllPages { OUTPUTS: for _, po := range p.pageOutputs { - if c := po.cp; c != nil { - if converted := c.convertedResult; converted != nil { - if hasIdentify(converted) { - c.Reset() - p.forceRender = true - continue OUTPUTS - } + if po.cp == nil { + continue + } + for id, _ := range idset { + if po.cp.dependencyTracker.Search(id) != nil { + po.cp.Reset() + p.forceRender = true + continue OUTPUTS } } } for _, s := range p.shortcodeState.shortcodes { - for _, id := range ids { - if s.info.Search(id.GetIdentity()) != nil { + for id, _ := range idset { + if s.info.Search(id) != nil { for _, po := range p.pageOutputs { if po.cp != nil { po.cp.Reset() diff --git a/hugolib/page.go b/hugolib/page.go index ce28988fff0..d9695fbe7c1 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -567,6 +567,7 @@ func (p *pageState) AlternativeOutputFormats() page.OutputFormats { return o } +// TODO1 option: map: display: inline/block (default inline) func (p *pageState) RenderString(in interface{}) (template.HTML, error) { s, err := cast.ToStringE(in) if err != nil { diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 91d5db8d9c8..e94fcb88d3f 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -23,6 +23,8 @@ import ( "sync" "unicode/utf8" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/converter" @@ -60,14 +62,22 @@ var ( } ) +var pageContentOutputDependenciesID = identity.KeyValueIdentity{Key: "pageOutput", Value: "dependencies"} + func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput, error) { parent := p.init return func(po *pageOutput) (*pageContentOutput, error) { + var dependencyTracker identity.Manager + if p.s.running() { + dependencyTracker = identity.NewIdentityManager(pageContentOutputDependenciesID) + } + cp := &pageContentOutput{ - p: p, - f: po.f, + dependencyTracker: dependencyTracker, + p: p, + f: po.f, } initContent := func() (err error) { @@ -114,7 +124,8 @@ func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput if err != nil { return err } - cp.convertedResult = r + + cp.convertedResult = r // TODO1 avoid storing this cp.workContent = r.Bytes() if _, ok := r.(converter.TableOfContentsProvider); !ok { @@ -238,8 +249,9 @@ type pageContentOutput struct { // Content state - workContent []byte - convertedResult converter.Result + workContent []byte + convertedResult converter.Result + dependencyTracker identity.Manager // Set in server mode. // Temporary storage of placeholders mapped to their content. // These are shortcodes etc. Some of these will need to be replaced @@ -260,7 +272,16 @@ type pageContentOutput struct { readingTime int } +func (p *pageContentOutput) trackDependency(id identity.Provider) { + if p.dependencyTracker != nil { + p.dependencyTracker.Add(id) + } +} + func (p *pageContentOutput) Reset() { + if p.dependencyTracker != nil { + p.dependencyTracker.Reset() + } p.p.initOutputFormats() p.initMain.Reset() p.initPlain.Reset() @@ -351,12 +372,23 @@ func (p *pageContentOutput) setAutoSummary() error { func (cp *pageContentOutput) renderContent(content []byte, renderTOC bool) (converter.Result, error) { c := cp.p.getContentConverter() - return c.Convert( + r, err := c.Convert( converter.RenderContext{ Src: content, RenderTOC: renderTOC, RenderHooks: cp.renderHooks, }) + + if err == nil { + if ids, ok := r.(identity.IdentitiesProvider); ok { + for _, v := range ids.GetIdentities() { + cp.trackDependency(v) + } + } + } + + return r, err + } func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) { diff --git a/hugolib/site.go b/hugolib/site.go index be1d178ba6d..7da1ed5dbcf 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -894,7 +894,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro events = s.filterFileEvents(events) events = s.translateFileEvents(events) - var changeIdentities identity.Identities + changeIdentities := make(identity.Identities) s.Log.DEBUG.Printf("Rebuild for events %q", events) @@ -922,7 +922,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro for _, ev := range events { id, found := s.eventToIdentity(ev) if found { - changeIdentities = append(changeIdentities, id) + changeIdentities[id] = id if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) diff --git a/identity/identity.go b/identity/identity.go index d7be4ebba66..184254a5649 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -6,62 +6,42 @@ import ( "sync" ) -// NewIdentityManager creates a new Manager starting at root. -func NewIdentityManager(root Provider) Manager { +// NewIdentityManager creates a new Manager starting at id. +func NewIdentityManager(id Provider) Manager { return &identityManager{ - Provider: root, - children: make(Identities, 0), + Provider: id, + ids: Identities{id.GetIdentity(): id}, } } // NewPathIdentity creates a new Identity with the two identifiers // type and path. -func NewPathIdentity(typ, path string) PathIdentity { - path = strings.TrimPrefix(filepath.ToSlash(path), "/") - return PathIdentity{Type: typ, Path: path} +func NewPathIdentity(typ, pat string) PathIdentity { + pat = strings.TrimPrefix(filepath.ToSlash(pat), "/") + return PathIdentity{Type: typ, Path: pat} } // Identities stores identity providers. -type Identities []Provider - -// A set of identities. -type IdentitiesSet map[Identity]bool - -// ToIdentitySet creates a set of these Identities. -func (ids Identities) ToIdentitySet() map[Identity]bool { - m := make(map[Identity]bool) - for _, id := range ids { - m[id.GetIdentity()] = true - } - return m -} +type Identities map[Identity]Provider func (ids Identities) search(id Identity) Provider { + if v, found := ids[id]; found { + return v + } for _, v := range ids { - vid := v.GetIdentity() - - if vid == id { - return v - } - - if idsp, ok := v.(ChildIdentitiesProvider); ok { - if nested := idsp.GetChildIdentities().search(id); nested != nil { + switch t := v.(type) { + case IdentitiesProvider: + if nested := t.GetIdentities().search(id); nested != nil { return nested } } } - return nil } -// IdentitiesProvider provides Identities as a set. +// IdentitiesProvider provides all Identities. type IdentitiesProvider interface { - GetIdentities() IdentitiesSet -} - -// ChildIdentitiesProvider provides child Identities. -type ChildIdentitiesProvider interface { - GetChildIdentities() Identities + GetIdentities() Identities } // Identity represents an thing that can provide an identify. This can be @@ -73,10 +53,11 @@ type Identity interface { // Manager manages identities, and is itself a Provider of Identity. type Manager interface { - ChildIdentitiesProvider + IdentitiesProvider Provider Add(ids ...Provider) Search(id Identity) Provider + Reset() } // A PathIdentity is a common identity identified by a type and a path, e.g. "layouts" and "_default/single.html". @@ -95,34 +76,56 @@ func (id PathIdentity) Name() string { return id.Path } +// A KeyValueIdentity a general purpose identity. +type KeyValueIdentity struct { + Key string + Value string +} + +// GetIdentity returns itself. +func (id KeyValueIdentity) GetIdentity() Identity { + return id +} + +// Name returns the Key. +func (id KeyValueIdentity) Name() string { + return id.Key +} + // Provider provides the hashable Identity. type Provider interface { GetIdentity() Identity } type identityManager struct { - sync.RWMutex + sync.Mutex Provider - children Identities + ids Identities } func (im *identityManager) Add(ids ...Provider) { im.Lock() - im.children = append(im.children, ids...) + for _, id := range ids { + im.ids[id.GetIdentity()] = id + } im.Unlock() } -func (im *identityManager) GetChildIdentities() Identities { - return im.children +func (im *identityManager) Reset() { + im.Lock() + id := im.GetIdentity() + im.ids = Identities{id.GetIdentity(): id} + im.Unlock() } -func (im *identityManager) Search(id Identity) Provider { - im.RLock() - defer im.RUnlock() - if id == im.GetIdentity() { - return im - } - v := im.children.search(id) +func (im *identityManager) GetIdentities() Identities { + im.Lock() + defer im.Unlock() + return im.ids +} - return v +func (im *identityManager) Search(id Identity) Provider { + im.Lock() + defer im.Unlock() + return im.ids.search(id.GetIdentity()) } diff --git a/magefile.go b/magefile.go index a888fb78d9c..38e8e057d0d 100644 --- a/magefile.go +++ b/magefile.go @@ -320,7 +320,7 @@ func runCmd(env map[string]string, cmd string, args ...string) error { } func isGoLatest() bool { - return strings.Contains(runtime.Version(), "1.12") + return strings.Contains(runtime.Version(), "1.13") } func isCI() bool { diff --git a/markup/converter/converter.go b/markup/converter/converter.go index dd7b4a347d8..a4585bd0380 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -105,8 +105,8 @@ type DocumentContext struct { // RenderContext holds contextual information about the content to render. type RenderContext struct { - Src []byte - RenderTOC bool + Src []byte + RenderTOC bool RenderHooks *hooks.Render } diff --git a/markup/converter/hooks/hooks.go b/markup/converter/hooks/hooks.go index c938753a160..15eec4fac27 100644 --- a/markup/converter/hooks/hooks.go +++ b/markup/converter/hooks/hooks.go @@ -24,6 +24,7 @@ type LinkContext interface { Destination() string Title() string Text() string + Resolved() bool // TODO1 consider } type Render struct { @@ -31,15 +32,6 @@ type Render struct { ImageRenderer LinkRenderer } -func (h *Render) GetChildIdentities() identity.Identities { - var ids identity.Identities - if h.LinkRenderer != nil { - ids = append(ids, h.LinkRenderer) - } - return ids - -} - type LinkRenderer interface { Render(w io.Writer, ctx LinkContext) error identity.Provider diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index 1586acbb34d..3633fff013e 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -151,14 +151,14 @@ var _ identity.IdentitiesProvider = (*converterResult)(nil) type converterResult struct { converter.Result toc tableofcontents.Root - ids identity.IdentitiesSet + ids identity.Identities } func (c converterResult) TableOfContents() tableofcontents.Root { return c.toc } -func (c converterResult) GetIdentities() identity.IdentitiesSet { +func (c converterResult) GetIdentities() identity.Identities { return c.ids } @@ -176,7 +176,7 @@ type renderContextData interface { type renderContextDataHolder struct { rctx converter.RenderContext dctx converter.DocumentContext - ids map[identity.Identity]bool + ids identity.Manager } func (ctx *renderContextDataHolder) RenderContext() converter.RenderContext { @@ -188,11 +188,11 @@ func (ctx *renderContextDataHolder) DocumentContext() converter.DocumentContext } func (ctx *renderContextDataHolder) AddIdentity(id identity.Identity) { - if _, found := ctx.ids[id]; !found { - ctx.ids[id] = true - } + ctx.ids.Add(id) } +var goldmarkConverterIdentity = identity.KeyValueIdentity{Key: "goldmark", Value: "converter"} + func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result converter.Result, err error) { defer func() { if r := recover(); r != nil { @@ -218,7 +218,7 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert rcx := &renderContextDataHolder{ rctx: ctx, dctx: c.ctx, - ids: make(map[identity.Identity]bool), + ids: identity.NewIdentityManager(goldmarkConverterIdentity), } w := renderContext{ @@ -232,7 +232,7 @@ func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result convert return converterResult{ Result: buf, - ids: rcx.ids, + ids: rcx.ids.GetIdentities(), toc: pctx.TableOfContents(), }, nil diff --git a/markup/goldmark/render_link.go b/markup/goldmark/render_link.go index d78829050c5..17ba5badacc 100644 --- a/markup/goldmark/render_link.go +++ b/markup/goldmark/render_link.go @@ -49,6 +49,10 @@ func (ctx linkContext) Destination() string { return ctx.destination } +func (ctx linkContext) Resolved() bool { + return false +} + func (ctx linkContext) Page() interface{} { return ctx.page } diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 69ca2d800b4..f440c4a2466 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -200,10 +200,18 @@ func (t *templateHandler) Lookup(name string) (tpl.Template, bool) { } +func (t *templateHandler) getTemplateInfo(name string) (tpl.Info, bool) { + t.mu.Lock() + defer t.mu.Unlock() + info, found := t.templateInfo[name] + return info, found + +} + func (t *templateHandler) applyTemplateInfo(templ tpl.Template, found bool) (tpl.Template, bool) { if adapter, ok := templ.(*tpl.TemplateAdapter); ok { if adapter.Info.IsZero() { - if info, found := t.templateInfo[templ.Name()]; found { + if info, found := t.getTemplateInfo(templ.Name()); found { adapter.Info = info } } @@ -618,6 +626,9 @@ func (t *templateHandler) addTemplate(name, tpl string) error { } func (t *templateHandler) getOrCreateTemplateInfo(name string) tpl.Info { + t.mu.Lock() + defer t.mu.Unlock() + info, found := t.templateInfo[name] if found { return info From c59d053f629d82c9bd876d544dcad57d8915ddcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 6 Dec 2019 11:30:16 +0100 Subject: [PATCH 04/19] Work --- hugolib/page.go | 7 ++++--- hugolib/site.go | 6 +++--- tpl/template.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/hugolib/page.go b/hugolib/page.go index d9695fbe7c1..3340b6fb6ab 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -380,15 +380,16 @@ func (p *pageState) createRenderHooks(f output.Format) (*hooks.Render, error) { layoutDescriptor := p.getLayoutDescriptor() layoutDescriptor.RenderingHook = true layoutDescriptor.LayoutOverride = false + layoutDescriptor.Layout = "" layoutDescriptor.Kind = "render-link" - linkLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + linkLayouts, err := p.s.templateLookupHandler.LayoutHandler.For(layoutDescriptor, f) if err != nil { return nil, err } layoutDescriptor.Kind = "render-image" - imageLayouts, err := p.s.layoutHandler.For(layoutDescriptor, f) + imageLayouts, err := p.s.templateLookupHandler.LayoutHandler.For(layoutDescriptor, f) if err != nil { return nil, err } @@ -466,7 +467,7 @@ func (p *pageState) getLayouts(layouts ...string) ([]string, error) { layoutDescriptor.LayoutOverride = true } - return p.s.layoutHandler.For(layoutDescriptor, f) + return p.s.templateLookupHandler.LayoutHandler.For(layoutDescriptor, f) } // This is serialized diff --git a/hugolib/site.go b/hugolib/site.go index 7da1ed5dbcf..ad7c36b6660 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -104,7 +104,7 @@ type Site struct { Sections Taxonomy Info SiteInfo - layoutHandler *output.LayoutHandler + templateLookupHandler *tpl.TemplateLookupHandler language *langs.Language @@ -323,7 +323,7 @@ func (s *Site) isEnabled(kind string) bool { // reset returns a new Site prepared for rebuild. func (s *Site) reset() *Site { return &Site{Deps: s.Deps, - layoutHandler: output.NewLayoutHandler(), + templateLookupHandler: tpl.NewTemplateLookupHandler(), disabledKinds: s.disabledKinds, titleFunc: s.titleFunc, relatedDocsHandler: s.relatedDocsHandler.Clone(), @@ -438,7 +438,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { s := &Site{ PageCollections: c, - layoutHandler: output.NewLayoutHandler(), + templateLookupHandler: tpl.NewTemplateLookupHandler(), language: cfg.Language, disabledKinds: disabledKinds, titleFunc: titleFunc, diff --git a/tpl/template.go b/tpl/template.go index 0d7598fde96..a232a2fa635 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -19,6 +19,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "time" "github.com/gohugoio/hugo/output" @@ -304,3 +305,53 @@ type TemplateFuncsGetter interface { type TemplateTestMocker interface { SetFuncs(funcMap map[string]interface{}) } + +type TemplateLookupHandler struct { + mu sync.RWMutex + cache map[layoutCacheKey]Template + LayoutHandler *output.LayoutHandler +} + +func NewTemplateLookupHandler() *TemplateLookupHandler { + return &TemplateLookupHandler{ + LayoutHandler: output.NewLayoutHandler(), + cache: make(map[layoutCacheKey]Template), + } +} + +func (t *TemplateLookupHandler) GetOrLookup(d output.LayoutDescriptor, f output.Format, lookup func(name string) (Template, bool)) (templ Template, found bool, err error) { + key := layoutCacheKey{d: d, f: f.Name} + + t.mu.RLock() + templ, found = t.cache[key] + t.mu.RUnlock() + + if found { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + var layouts []string + layouts, err = t.LayoutHandler.For(d, f) + if err != nil { + return + } + + for _, l := range layouts { + templ, found = lookup(l) + if found { + t.cache[key] = templ + } + return + } + + return + +} + +type layoutCacheKey struct { + d output.LayoutDescriptor + f string +} From a8ef747f6f42733512de623eb561338977f0ea4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 6 Dec 2019 12:14:22 +0100 Subject: [PATCH 05/19] Perf --- hugolib/page.go | 5 +++++ hugolib/page__new.go | 1 + hugolib/page__output.go | 27 +++++++++++++++++++++++++++ hugolib/page__per_output.go | 6 +++++- markup/converter/hooks/hooks.go | 20 ++++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/hugolib/page.go b/hugolib/page.go index 3340b6fb6ab..b00c3cbb91d 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -328,12 +328,17 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error { return nil } +// TODO1 remove func (ps *pageState) initOutputFormats() error { if len(ps.pageOutputs) == 0 { return nil } + if true { + return nil + } + c := ps.getContentConverter() if c == nil || !c.Supports(converter.FeatureRenderHooks) { return nil diff --git a/hugolib/page__new.go b/hugolib/page__new.go index 37db148f8bd..902f07d867d 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -246,6 +246,7 @@ func newPageWithContent(f *fileInfo, s *Site, bundled bool, content resource.Ope } ps.init.Add(func() (interface{}, error) { + // TODO1 reuseContent := ps.renderable && !ps.shortcodeState.hasShortcodes() // Creates what's needed for each output format. diff --git a/hugolib/page__output.go b/hugolib/page__output.go index b07dbc9126d..0afc6859317 100644 --- a/hugolib/page__output.go +++ b/hugolib/page__output.go @@ -14,6 +14,7 @@ package hugolib import ( + "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" @@ -90,6 +91,32 @@ type pageOutput struct { cp *pageContentOutput } +func (o *pageOutput) initRenderHooks() error { + if !o.render || o.cp == nil { + return nil + } + + ps := o.cp.p + + c := ps.getContentConverter() + if c == nil || !c.Supports(converter.FeatureRenderHooks) { + return nil + } + + h, err := ps.createRenderHooks(o.f) + if err != nil { + return err + } + if h == nil { + return nil + } + + o.cp.renderHooks = h + + return nil + +} + func (p *pageOutput) initContentProvider(cp *pageContentOutput) { if cp == nil { return diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index e94fcb88d3f..88436f112f8 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -93,6 +93,10 @@ func newPageContentOutput(p *pageState) func(po *pageOutput) (*pageContentOutput } }() + if err := po.initRenderHooks(); err != nil { + return err + } + var hasVariants bool f := po.f @@ -245,7 +249,7 @@ type pageContentOutput struct { // May be nil. renderHooks *hooks.Render // Set if there are more than one output format variant - renderHooksHaveVariants bool + renderHooksHaveVariants bool // TODO1 reimplement this in another way // Content state diff --git a/markup/converter/hooks/hooks.go b/markup/converter/hooks/hooks.go index 15eec4fac27..3be1fd1c3c6 100644 --- a/markup/converter/hooks/hooks.go +++ b/markup/converter/hooks/hooks.go @@ -32,6 +32,26 @@ type Render struct { ImageRenderer LinkRenderer } +func (r *Render) Eq(other interface{}) bool { + ro, ok := other.(*Render) + if !ok { + return false + } + if r == nil || ro == nil { + return r == nil + } + + if r.ImageRenderer.GetIdentity() != ro.ImageRenderer.GetIdentity() { + return false + } + + if r.LinkRenderer.GetIdentity() != ro.LinkRenderer.GetIdentity() { + return false + } + + return true +} + type LinkRenderer interface { Render(w io.Writer, ctx LinkContext) error identity.Provider From ddd8c6598764c6fb8468b35282d132cd8310ab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 6 Dec 2019 14:33:06 +0100 Subject: [PATCH 06/19] Bench --- hugolib/site_benchmark_new_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/hugolib/site_benchmark_new_test.go b/hugolib/site_benchmark_new_test.go index 646124b09ca..13302300ee9 100644 --- a/hugolib/site_benchmark_new_test.go +++ b/hugolib/site_benchmark_new_test.go @@ -127,6 +127,36 @@ title = "What is Markdown" baseURL = "https://example.com" `) + + data, err := ioutil.ReadFile(filepath.FromSlash("testdata/what-is-markdown.md")) + sb.Assert(err, qt.IsNil) + datastr := string(data) + getContent := func(i int) string { + return fmt.Sprintf(`--- +title: "Page %d" +--- + +`, i) + datastr + + } + for i := 1; i <= 100; i++ { + sb.WithContent(fmt.Sprintf("content/page%d.md", i), getContent(i)) + } + + return sb + }, + func(s *sitesBuilder) { + s.Assert(s.CheckExists("public/page8/index.html"), qt.Equals, true) + }, + }, + {"Markdown with custom link handler", func(b testing.TB) *sitesBuilder { + sb := newTestSitesBuilder(b).WithConfigFile("toml", ` +title = "What is Markdown" +baseURL = "https://example.com" + +`) + + sb.WithTemplatesAdded("_default/_markup/render-link.html", `CUSTOM LINK`) data, err := ioutil.ReadFile(filepath.FromSlash("testdata/what-is-markdown.md")) sb.Assert(err, qt.IsNil) datastr := string(data) From 835ed84c2475c69b73736b0bb828068e3389d818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 6 Dec 2019 18:21:19 +0100 Subject: [PATCH 07/19] Paritals --- hugolib/template_test.go | 34 ++++++++++++++++++++ tpl/tplimpl/template_ast_transformers.go | 40 +++++++++++++++--------- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 71b4b46c0bf..63832c502bb 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -18,6 +18,10 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/tpl" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" @@ -331,3 +335,33 @@ Partial cached3: {{ partialCached "p1" "input3" $key2 }} Partial cached3: partial: input3 `) } + +func TestTemplateDependencies(t *testing.T) { + b := newTestSitesBuilder(t) + + b.WithTemplates("index.html", ` +{{ partial "p1.html" }} +{{ partialCached "p2.html" "foo" }} +{{ partials.Include "p3.html" }} +{{ partials.IncludeCached "p4.html" "foo" }} +`, + "partials/p1.html", `p1`, + "partials/p2.html", `p2`, + "partials/p3.html", `p3`, + "partials/p4.html", `p4`, + ) + + b.WithContent("p1.md", "") + + b.Build(BuildCfg{}) + + s := b.H.Sites[0] + + templ, found := s.lookupTemplate("index.html") + b.Assert(found, qt.Equals, true) + + ids := templ.(tpl.TemplateInfoProvider).TemplateInfo().GetIdentities() + + b.Assert(ids, qt.HasLen, 23) + +} diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 5e11db78b05..18e030d1c6f 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -15,6 +15,7 @@ package tplimpl import ( "html/template" + "regexp" "strings" texttemplate "text/template" "text/template/parse" @@ -229,6 +230,8 @@ func (c *templateContext) wrapWithGetIf(p *parse.PipeNode) { } +var partialRe = regexp.MustCompile(`^partial(Cached)?$|^partials\.Include(Cached)?$`) + // applyTransformations do 3 things: // 1) Make all .Params.CamelCase and similar into lowercase. // 2) Wraps every with and if pipe in getif @@ -270,21 +273,28 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { case *parse.CommandNode: if len(x.Args) > 1 { - if id, ok := x.Args[0].(*parse.IdentifierNode); ok { - if id.Ident == "partial" { - partialName := strings.Trim(x.Args[1].String(), "\"") - if !strings.Contains(partialName, ".") { - partialName += ".html" - } - // TODO1 add a test for case - partialName = "partials/" + partialName - info := c.lookupFn(partialName) - if info != nil { - c.Info.Add(info.info) - } else { - // Delay for later - c.identityNotFound[partialName] = true - } + first := x.Args[0] + var id string + switch v := first.(type) { + case *parse.IdentifierNode: + id = v.Ident + case *parse.ChainNode: + id = v.String() + } + + if partialRe.MatchString(id) { + partialName := strings.Trim(x.Args[1].String(), "\"") + if !strings.Contains(partialName, ".") { + partialName += ".html" + } + // TODO1 add a test for case + partialName = "partials/" + partialName + info := c.lookupFn(partialName) + if info != nil { + c.Info.Add(info.info) + } else { + // Delay for later + c.identityNotFound[partialName] = true } } } From 62306375a0416390ae4148af7ccbe4742d5457fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 7 Dec 2019 11:09:17 +0100 Subject: [PATCH 08/19] Work --- hugolib/template_test.go | 29 ++++++++++++++++++++---- identity/identity.go | 2 +- tpl/tplimpl/template.go | 3 ++- tpl/tplimpl/template_ast_transformers.go | 6 +++-- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 63832c502bb..b58f812cedb 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -340,18 +340,34 @@ func TestTemplateDependencies(t *testing.T) { b := newTestSitesBuilder(t) b.WithTemplates("index.html", ` -{{ partial "p1.html" }} +{{ $p := site.GetPage "p1" }} +{{ partial "p1.html" $p }} {{ partialCached "p2.html" "foo" }} -{{ partials.Include "p3.html" }} +{{ partials.Include "p3.html" "data" }} {{ partials.IncludeCached "p4.html" "foo" }} +{{ $p := partial "p5" }} +{{ partial "sub/p6.html" }} +{{ partial "P7.html" }} +{{ template "_default/foo.html" }} + `, - "partials/p1.html", `p1`, + "partials/p1.html", `ps: {{ .Render "li" }}`, "partials/p2.html", `p2`, "partials/p3.html", `p3`, "partials/p4.html", `p4`, + "partials/p5.html", `p5`, + "partials/sub/p6.html", `p6`, + "partials/P7.html", `p7`, + "_default/foo.html", `foo`, + "_default/li.html", `li`, ) - b.WithContent("p1.md", "") + b.WithContent("p1.md", `--- +title: P1 +--- + + +`) b.Build(BuildCfg{}) @@ -362,6 +378,9 @@ func TestTemplateDependencies(t *testing.T) { ids := templ.(tpl.TemplateInfoProvider).TemplateInfo().GetIdentities() - b.Assert(ids, qt.HasLen, 23) + //b.AssertFileContent("public/index.html", `FOO`) + // TODO1 .Render... + + b.Assert(ids, qt.HasLen, 9) } diff --git a/identity/identity.go b/identity/identity.go index 184254a5649..bf03189cc9e 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -17,7 +17,7 @@ func NewIdentityManager(id Provider) Manager { // NewPathIdentity creates a new Identity with the two identifiers // type and path. func NewPathIdentity(typ, pat string) PathIdentity { - pat = strings.TrimPrefix(filepath.ToSlash(pat), "/") + pat = strings.ToLower(strings.TrimPrefix(filepath.ToSlash(pat), "/")) return PathIdentity{Type: typ, Path: pat} } diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index f440c4a2466..0795ae59106 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -528,6 +528,7 @@ func (t *htmlTemplates) addTemplateIn(tt *template.Template, name, tpl string) ( for k := range c.templateNotFound { t.transformNotFound[k] = true + t.identityNotFound[k] = append(t.identityNotFound[k], c.Info) } for k := range c.identityNotFound { @@ -695,7 +696,7 @@ func (t *templateHandler) postTransform() error { for k, v := range s.identityNotFound { templ := s.lookup(k) - if templ != nil { + if templ != nil && templ.info.Manager != nil { id := templ.info.GetIdentity() for _, im := range v { im.Add(id) diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 18e030d1c6f..5b29701d322 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -80,8 +80,11 @@ func (c templateContext) getIfNotVisited(name string) *templateInfoTree { // and not yet parsed. Unusual, but it happens. // Store the name to try again later. c.templateNotFound[name] = true + } else { + if templ.info.Manager != nil { + c.Info.Add(templ.info) + } } - return templ } @@ -287,7 +290,6 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { if !strings.Contains(partialName, ".") { partialName += ".html" } - // TODO1 add a test for case partialName = "partials/" + partialName info := c.lookupFn(partialName) if info != nil { From 0fdaff9222454b5d486fb9715c5bdd1b9d12e868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 7 Dec 2019 13:06:20 +0100 Subject: [PATCH 09/19] Work --- hugolib/page.go | 1 + hugolib/template_test.go | 7 ++-- tpl/compare/init.go | 7 ++++ tpl/compare/truth.go | 6 ++++ tpl/tplimpl/template_ast_transformers.go | 44 ++++++++++++++++++++++-- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/hugolib/page.go b/hugolib/page.go index b00c3cbb91d..cad0497e09c 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -592,6 +592,7 @@ func (p *pageState) RenderString(in interface{}) (template.HTML, error) { } func (p *pageState) Render(layout ...string) (template.HTML, error) { + fmt.Println("RENDER:", layout) l, err := p.getLayouts(layout...) if err != nil { return "", p.wrapError(errors.Errorf("failed to resolve layout %v", layout)) diff --git a/hugolib/template_test.go b/hugolib/template_test.go index b58f812cedb..99ad648544e 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -340,9 +340,10 @@ func TestTemplateDependencies(t *testing.T) { b := newTestSitesBuilder(t) b.WithTemplates("index.html", ` +{{ $m := dict "Render" "foo" }} {{ $p := site.GetPage "p1" }} {{ partial "p1.html" $p }} -{{ partialCached "p2.html" "foo" }} +{{ partialCached "p2.html" $m }} {{ partials.Include "p3.html" "data" }} {{ partials.IncludeCached "p4.html" "foo" }} {{ $p := partial "p5" }} @@ -351,8 +352,8 @@ func TestTemplateDependencies(t *testing.T) { {{ template "_default/foo.html" }} `, - "partials/p1.html", `ps: {{ .Render "li" }}`, - "partials/p2.html", `p2`, + "partials/p1.html", `p1: {{ .Render "li" }}{{ .Render }}`, + "partials/p2.html", `p2: {{ .Render }} `, "partials/p3.html", `p3`, "partials/p4.html", `p4`, "partials/p5.html", `p5`, diff --git a/tpl/compare/init.go b/tpl/compare/init.go index 2e536ff04ab..60ff9c3e28b 100644 --- a/tpl/compare/init.go +++ b/tpl/compare/init.go @@ -87,6 +87,13 @@ func init() { [][2]string{}, ) + ns.AddMethodMapping(ctx.invokeDot, + []string{"invokeDot"}, + [][2]string{}, + ) + + // + ns.AddMethodMapping(ctx.Not, []string{"not"}, [][2]string{}, diff --git a/tpl/compare/truth.go b/tpl/compare/truth.go index 85ee22121e9..bec78980e2a 100644 --- a/tpl/compare/truth.go +++ b/tpl/compare/truth.go @@ -17,6 +17,7 @@ package compare import ( + "fmt" "reflect" "github.com/gohugoio/hugo/common/hreflect" @@ -37,6 +38,11 @@ func (*Namespace) getIf(arg reflect.Value) reflect.Value { return reflect.ValueOf("") } +func (*Namespace) invokeDot(args ...interface{}) interface{} { + fmt.Println("invokeDot:", args) + return "FOO" +} + // And computes the Boolean AND of its arguments, returning // the first false argument it encounters, or the last argument. func (*Namespace) And(arg0 reflect.Value, args ...reflect.Value) reflect.Value { diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 5b29701d322..9d7978d31f8 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -181,16 +181,31 @@ func applyTemplateTransformers( const ( partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}` + dotContextWrapperTempl = `{{ invokeDot . "NAME" "ARGS" }}` ) -var partialReturnWrapper *parse.ListNode +var ( + partialReturnWrapper *parse.ListNode + dotContextWrapper *parse.CommandNode +) func init() { - templ, err := texttemplate.New("").Parse(partialReturnWrapperTempl) + tt := texttemplate.New("").Funcs(texttemplate.FuncMap{ + "invokeDot": func() interface{} { return "foo" }, + }) + + templ, err := tt.Parse(partialReturnWrapperTempl) if err != nil { panic(err) } partialReturnWrapper = templ.Tree.Root + + templ, err = tt.Parse(dotContextWrapperTempl) + if err != nil { + panic(err) + } + action := templ.Tree.Root.Nodes[0].(*parse.ActionNode) + dotContextWrapper = action.Pipe.Cmds[0] } func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.ListNode { @@ -208,6 +223,28 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L } +func (c *templateContext) wrapDot(n *parse.CommandNode) bool { + field, ok := n.Args[0].(*parse.FieldNode) + if !ok { + return false + } + + wrapper := dotContextWrapper.Copy().(*parse.CommandNode) + + sn := wrapper.Args[2].(*parse.StringNode) + fields := field.String() + sn.Quoted = "\"" + fields + "\"" + sn.Text = fields + + args := wrapper.Args[:3] + if len(n.Args) > 2 { + args = append(args, n.Args[1:]...) + } + n.Args = args + return true + +} + // The truth logic in Go's template package is broken for certain values // for the if and with keywords. This works around that problem by wrapping // the node passed to if/with in a getif conditional. @@ -275,7 +312,8 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: - if len(x.Args) > 1 { + if c.wrapDot(x) { + } else if len(x.Args) > 1 { first := x.Args[0] var id string switch v := first.(type) { From d16e6255bb1fbfe369badcb0af8615b2d9a62008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 Dec 2019 11:22:17 +0100 Subject: [PATCH 10/19] Dot --- common/hreflect/helpers.go | 13 +- common/hreflect/invoke.go | 148 +++++++++++++++++++++++ common/hreflect/invoke_test.go | 107 ++++++++++++++++ hugolib/page.go | 1 - langs/language.go | 2 +- tpl/compare/truth.go | 17 ++- tpl/tplimpl/template_ast_transformers.go | 7 +- 7 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 common/hreflect/invoke.go create mode 100644 common/hreflect/invoke_test.go diff --git a/common/hreflect/helpers.go b/common/hreflect/helpers.go index db7b208b5b6..eace5b3d353 100644 --- a/common/hreflect/helpers.go +++ b/common/hreflect/helpers.go @@ -35,6 +35,7 @@ func IsTruthful(in interface{}) bool { } var zeroType = reflect.TypeOf((*types.Zeroer)(nil)).Elem() +var zero reflect.Value // IsTruthfulValue returns whether the given value has a meaningful truth value. // This is based on template.IsTrue in Go's stdlib, but also considers @@ -79,7 +80,8 @@ func IsTruthfulValue(val reflect.Value) (truth bool) { return } -// Based on: https://github.com/golang/go/blob/178a2c42254166cffed1b25fb1d3c7a5727cada6/src/text/template/exec.go#L931 +// Indirect funcs below borrowed from Go: +// https://github.com/golang/go/blob/178a2c42254166cffed1b25fb1d3c7a5727cada6/src/text/template/exec.go#L931 func indirectInterface(v reflect.Value) reflect.Value { if v.Kind() != reflect.Interface { return v @@ -89,3 +91,12 @@ func indirectInterface(v reflect.Value) reflect.Value { } return v.Elem() } + +func indirect(v reflect.Value) (rv reflect.Value, isNil bool) { + for ; v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface; v = v.Elem() { + if v.IsNil() { + return v, true + } + } + return v, false +} diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go new file mode 100644 index 00000000000..3d9d1ec0edc --- /dev/null +++ b/common/hreflect/invoke.go @@ -0,0 +1,148 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// Some functions in this file (see comments) is based on the Go source code, +// copyright The Go Authors and governed by a BSD-style license. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hreflect + +import ( + "fmt" + "reflect" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/pkg/errors" +) + +var ( + errorType = reflect.TypeOf((*error)(nil)).Elem() + fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() + reflectValueType = reflect.TypeOf((*reflect.Value)(nil)).Elem() +) + +func Invoke(dot interface{}, path []string, args ...interface{}) (interface{}, error) { + v := reflect.ValueOf(dot) + result, err := invoke(v, path, args) + if err != nil { + return nil, err + } + + if !result.IsValid() { + return nil, nil + } + + return result.Interface(), nil + +} + +func argsToValues(args []interface{}) []reflect.Value { + // TODO1 varargs + if len(args) == 0 { + return nil + } + argsv := make([]reflect.Value, len(args)) + for i, v := range args { + argsv[i] = reflect.ValueOf(v) + } + return argsv +} + +func invoke(receiver reflect.Value, path []string, args []interface{}) (reflect.Value, error) { + if len(path) == 0 { + return receiver, nil + } + name := path[0] + nextPath := 1 + typ := receiver.Type() + receiver, isNil := indirect(receiver) + if receiver.Kind() == reflect.Interface && isNil { + return err("nil pointer evaluating %s.%s", typ, name) + } + + ptr := receiver + if ptr.Kind() != reflect.Interface && ptr.Kind() != reflect.Ptr && ptr.CanAddr() { + ptr = ptr.Addr() + } + + var fn reflect.Value + if typ.Kind() == reflect.Func { + fn = receiver + nextPath-- + } else { + fn = ptr.MethodByName(name) + } + + if fn.IsValid() { + mt := fn.Type() + if !isValidFunc(mt) { + return err("method %s not valid", name) + } + + var argsv []reflect.Value + if len(path) == 1 { + numArgs := len(args) + if mt.IsVariadic() { + if numArgs < (mt.NumIn() - 1) { + return err("methods %s expects at leas %d arguments, got %d", name, mt.NumIn()-1, numArgs) + } + } else if numArgs != mt.NumIn() { + return err("methods %s takes %d arguments, got %d", name, mt.NumIn(), numArgs) + } + argsv = argsToValues(args) + } + + result := fn.Call(argsv) + if mt.NumOut() == 2 { + if !result[1].IsZero() { + return reflect.Value{}, result[1].Interface().(error) + } + } + + return invoke(result[0], path[nextPath:], args) + } + + switch receiver.Kind() { + case reflect.Struct: + if f := receiver.FieldByName(name); f.IsValid() { + return invoke(f, path[1:], args) + } else { + return err("no method or field with name %s found", name) + } + case reflect.Map: + if p, ok := receiver.Interface().(maps.Params); ok { + // Do case insensitive map lookup + v := p.Get(path...) + return reflect.ValueOf(v), nil + } + v := receiver.MapIndex(reflect.ValueOf(name)) + if !v.IsValid() { + return reflect.Value{}, nil + } + return invoke(v, path[1:], args) + } + return receiver, nil +} + +func err(s string, args ...interface{}) (reflect.Value, error) { + return reflect.Value{}, errors.Errorf(s, args...) +} + +func isValidFunc(typ reflect.Type) bool { + switch { + case typ.NumOut() == 1: + return true + case typ.NumOut() == 2 && typ.Out(1) == errorType: + return true + } + return false +} diff --git a/common/hreflect/invoke_test.go b/common/hreflect/invoke_test.go new file mode 100644 index 00000000000..df0370eb8ff --- /dev/null +++ b/common/hreflect/invoke_test.go @@ -0,0 +1,107 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// Some functions in this file (see comments) is based on the Go source code, +// copyright The Go Authors and governed by a BSD-style license. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hreflect + +import ( + "testing" + + "github.com/gohugoio/hugo/common/maps" + + "github.com/pkg/errors" + + qt "github.com/frankban/quicktest" +) + +type testStruct struct { + Val string + Struct testStruct2 + err error +} + +type testStruct2 struct { + Val2 string + err error +} + +func (t testStruct) GetStruct() testStruct2 { + return t.Struct +} + +func (t testStruct) GetVal() string { + return t.Val +} + +func (t testStruct) OneArg(arg string) string { + return arg +} + +func (t *testStruct) GetValP() string { + return t.Val +} + +func (t testStruct) GetValError() (string, error) { + return t.Val, t.err +} + +func (t testStruct2) GetVal2() string { + return t.Val2 +} + +func (t testStruct2) GetVal2Error() (string, error) { + return t.Val2, t.err +} + +func TestInvoke(t *testing.T) { + c := qt.New(t) + + hello := testStruct{Val: "hello"} + + for _, test := range []struct { + name string + dot interface{} + path []string + args []interface{} + expect interface{} + }{ + {"Method", hello, []string{"GetVal"}, nil, "hello"}, + {"Method one arg", hello, []string{"OneArg"}, []interface{}{"hello"}, "hello"}, + {"Method pointer 1", &testStruct{Val: "hello"}, []string{"GetValP"}, nil, "hello"}, + {"Method pointer 2", &testStruct{Val: "hello"}, []string{"GetVal"}, nil, "hello"}, + {"Method error", testStruct{Val: "hello", err: errors.New("This failed")}, []string{"GetValError"}, nil, false}, + {"Method error nil", hello, []string{"GetValError"}, nil, "hello"}, + {"Func", func() testStruct { return hello }, []string{"GetVal"}, nil, "hello"}, + {"Func nested", func() testStruct { return testStruct{Val: "hello", Struct: testStruct2{Val2: "hello2"}} }, []string{"GetStruct", "GetVal2"}, nil, "hello2"}, + {"Field", hello, []string{"Val"}, nil, "hello"}, + {"Field pointer receiver", &testStruct{Val: "hello"}, []string{"Val"}, nil, "hello"}, + {"Method nested", testStruct{Val: "hello", Struct: testStruct2{Val2: "hello2"}}, []string{"GetStruct", "GetVal2"}, nil, "hello2"}, + {"Field nested", testStruct{Val: "hello", Struct: testStruct2{Val2: "hello2"}}, []string{"Struct", "Val2"}, nil, "hello2"}, + {"Method field nested", testStruct{Val: "hello", Struct: testStruct2{Val2: "hello2"}}, []string{"GetStruct", "Val2"}, nil, "hello2"}, + {"Method nested error", testStruct{Val: "hello", Struct: testStruct2{Val2: "hello2", err: errors.New("This failed")}}, []string{"GetStruct", "GetVal2Error"}, nil, false}, + {"Map", map[string]string{"hello": "world"}, []string{"hello"}, nil, "world"}, + {"Map not found", map[string]string{"hello": "world"}, []string{"Hugo"}, nil, nil}, // TODO1 nil type + {"Map nested", map[string]map[string]string{"hugo": map[string]string{"does": "rock"}}, []string{"hugo", "does"}, nil, "rock"}, + {"Params", maps.Params{"hello": "world"}, []string{"Hello"}, nil, "world"}, + } { + c.Run(test.name, func(c *qt.C) { + got, err := Invoke(test.dot, test.path, test.args...) + if b, ok := test.expect.(bool); ok && !b { + c.Assert(err, qt.Not(qt.IsNil)) + return + } + c.Assert(got, qt.DeepEquals, test.expect) + }) + } +} diff --git a/hugolib/page.go b/hugolib/page.go index cad0497e09c..b00c3cbb91d 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -592,7 +592,6 @@ func (p *pageState) RenderString(in interface{}) (template.HTML, error) { } func (p *pageState) Render(layout ...string) (template.HTML, error) { - fmt.Println("RENDER:", layout) l, err := p.getLayouts(layout...) if err != nil { return "", p.wrapError(errors.Errorf("failed to resolve layout %v", layout)) diff --git a/langs/language.go b/langs/language.go index 67cb3689a1d..2b9ed176065 100644 --- a/langs/language.go +++ b/langs/language.go @@ -123,7 +123,7 @@ func (l Languages) Less(i, j int) bool { func (l Languages) Swap(i, j int) { l[i], l[j] = l[j], l[i] } // Params retunrs language-specific params merged with the global params. -func (l *Language) Params() map[string]interface{} { +func (l *Language) Params() maps.Params { return l.params } diff --git a/tpl/compare/truth.go b/tpl/compare/truth.go index bec78980e2a..2277afbce42 100644 --- a/tpl/compare/truth.go +++ b/tpl/compare/truth.go @@ -17,8 +17,10 @@ package compare import ( - "fmt" "reflect" + "strings" + + "github.com/spf13/cast" "github.com/gohugoio/hugo/common/hreflect" ) @@ -38,9 +40,16 @@ func (*Namespace) getIf(arg reflect.Value) reflect.Value { return reflect.ValueOf("") } -func (*Namespace) invokeDot(args ...interface{}) interface{} { - fmt.Println("invokeDot:", args) - return "FOO" +func (*Namespace) invokeDot(in ...interface{}) (interface{}, error) { + dot := in[0] + path := strings.Split(cast.ToString(in[1])[1:], ".") + + var args []interface{} + if len(in) > 2 { + args = in[2:] + } + + return hreflect.Invoke(dot, path, args...) } // And computes the Boolean AND of its arguments, returning diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 9d7978d31f8..b785c713aba 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -237,10 +237,12 @@ func (c *templateContext) wrapDot(n *parse.CommandNode) bool { sn.Text = fields args := wrapper.Args[:3] - if len(n.Args) > 2 { + if len(n.Args) > 1 { args = append(args, n.Args[1:]...) } + n.Args = args + return true } @@ -370,6 +372,9 @@ func (c *templateContext) applyTransformationsToNodes(nodes ...parse.Node) { } func (c *templateContext) updateIdentsIfNeeded(idents []string) { + if true { + return // TODO1 remove all this .Params stuff. + } index := c.decl.indexOfReplacementStart(idents) if index == -1 { From 68965de508d3e5a21c8efb3ba7c398f3a0561f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 Dec 2019 17:04:51 +0100 Subject: [PATCH 11/19] Work --- common/hreflect/invoke.go | 45 +++++++++++++---- deps/deps.go | 10 ++-- hugolib/case_insensitive_test.go | 7 ++- hugolib/content_render_hooks_test.go | 30 ++++++++++++ hugolib/hugo_sites.go | 1 + tpl/collections/sort.go | 2 +- tpl/compare/compare.go | 23 ++++++++- tpl/compare/init.go | 2 +- tpl/compare/truth.go | 18 +++++-- tpl/template.go | 4 ++ tpl/tplimpl/template.go | 2 +- tpl/tplimpl/templateProvider.go | 7 ++- tpl/tplimpl/template_ast_transformers.go | 61 +++++++++++++++++++----- tpl/tplimpl/template_funcs.go | 3 +- 14 files changed, 178 insertions(+), 37 deletions(-) diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go index 3d9d1ec0edc..4aee1b4a53f 100644 --- a/common/hreflect/invoke.go +++ b/common/hreflect/invoke.go @@ -30,9 +30,35 @@ var ( reflectValueType = reflect.TypeOf((*reflect.Value)(nil)).Elem() ) -func Invoke(dot interface{}, path []string, args ...interface{}) (interface{}, error) { - v := reflect.ValueOf(dot) - result, err := invoke(v, path, args) +type Invoker struct { + funcs func(name string) interface{} +} + +func NewInvoker(funcs func(name string) interface{}) *Invoker { + return &Invoker{funcs: funcs} +} + +func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{}, error) { + name := path[0] + f := i.funcs(name) + if f == nil { + return err("function with name %s not found", name) + } + result, err := i.invoke(reflect.ValueOf(f), path, args) + if err != nil { + return nil, err + } + + if !result.IsValid() { + return nil, nil + } + + return result.Interface(), nil +} + +func (i *Invoker) InvokeMethod(receiver interface{}, path []string, args ...interface{}) (interface{}, error) { + v := reflect.ValueOf(receiver) + result, err := i.invoke(v, path, args) if err != nil { return nil, err } @@ -46,7 +72,6 @@ func Invoke(dot interface{}, path []string, args ...interface{}) (interface{}, e } func argsToValues(args []interface{}) []reflect.Value { - // TODO1 varargs if len(args) == 0 { return nil } @@ -57,11 +82,12 @@ func argsToValues(args []interface{}) []reflect.Value { return argsv } -func invoke(receiver reflect.Value, path []string, args []interface{}) (reflect.Value, error) { +func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface{}) (reflect.Value, error) { if len(path) == 0 { return receiver, nil } name := path[0] + nextPath := 1 typ := receiver.Type() receiver, isNil := indirect(receiver) @@ -77,7 +103,6 @@ func invoke(receiver reflect.Value, path []string, args []interface{}) (reflect. var fn reflect.Value if typ.Kind() == reflect.Func { fn = receiver - nextPath-- } else { fn = ptr.MethodByName(name) } @@ -108,15 +133,15 @@ func invoke(receiver reflect.Value, path []string, args []interface{}) (reflect. } } - return invoke(result[0], path[nextPath:], args) + return i.invoke(result[0], path[nextPath:], args) } switch receiver.Kind() { case reflect.Struct: if f := receiver.FieldByName(name); f.IsValid() { - return invoke(f, path[1:], args) + return i.invoke(f, path[1:], args) } else { - return err("no method or field with name %s found", name) + return err("no field with name %s found", name) } case reflect.Map: if p, ok := receiver.Interface().(maps.Params); ok { @@ -128,7 +153,7 @@ func invoke(receiver reflect.Value, path []string, args []interface{}) (reflect. if !v.IsValid() { return reflect.Value{}, nil } - return invoke(v, path[1:], args) + return i.invoke(v, path[1:], args) } return receiver, nil } diff --git a/deps/deps.go b/deps/deps.go index 4dc170a9713..4cc131d2931 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -40,6 +40,8 @@ type Deps struct { // The templates to use. This will usually implement the full tpl.TemplateHandler. Tmpl tpl.TemplateFinder `json:"-"` + TemplateFuncs map[string]interface{} `json:"-"` + // We use this to parse and execute ad-hoc text templates. TextTmpl tpl.TemplateParseFinder `json:"-"` @@ -76,7 +78,7 @@ type Deps struct { // All the output formats available for the current site. OutputFormatsConfig output.Formats - templateProvider ResourceProvider + TemplateProvider ResourceProvider WithTemplate func(templ tpl.TemplateHandler) error `json:"-"` translationProvider ResourceProvider @@ -162,7 +164,7 @@ func (d *Deps) LoadResources() error { return errors.Wrap(err, "loading translations") } - if err := d.templateProvider.Update(d); err != nil { + if err := d.TemplateProvider.Update(d); err != nil { return errors.Wrap(err, "loading templates") } @@ -243,7 +245,7 @@ func New(cfg DepsCfg) (*Deps, error) { Log: logger, DistinctErrorLog: distinctErrorLogger, DistinctWarningLog: distinctWarnLogger, - templateProvider: cfg.TemplateProvider, + TemplateProvider: cfg.TemplateProvider, translationProvider: cfg.TranslationProvider, WithTemplate: cfg.WithTemplate, PathSpec: ps, @@ -310,7 +312,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er return nil, err } - if err := d.templateProvider.Clone(&d); err != nil { + if err := d.TemplateProvider.Clone(&d); err != nil { return nil, err } diff --git a/hugolib/case_insensitive_test.go b/hugolib/case_insensitive_test.go index 4a6b8f3a06d..8844d31a512 100644 --- a/hugolib/case_insensitive_test.go +++ b/hugolib/case_insensitive_test.go @@ -259,7 +259,12 @@ func TestCaseInsensitiveConfigurationForAllTemplateEngines(t *testing.T) { {"html", noOp}, {"ace", noOp}, } { - doTestCaseInsensitiveConfigurationForTemplateEngine(t, config.suffix, config.templateFixer) + + config := config + t.Run(config.suffix, func(t *testing.T) { + t.Parallel() + doTestCaseInsensitiveConfigurationForTemplateEngine(t, config.suffix, config.templateFixer) + }) } diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index bcb18576e54..dcd412cdfce 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -15,6 +15,36 @@ package hugolib import "testing" +func TestTempT(t *testing.T) { + config := ` +baseURL="https://example.org" +defaultContentLanguageInSubDir=true + +[params] +[params.COLORS] +BLUE="nice" + +[languages] +[languages.en] +weight=1 +[languages.nn] +weight=2 +` + b := newTestSitesBuilder(t).WithConfigFile("toml", config) + b.WithTemplates( + "index.html", ` +{{ $params := .Site.Params }} +{{ $colors := $params.Colors }} +{{ $blue := $colors.Blue }} +Site en: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}|Blue: {{ $blue }}`, + "index.nn.html", `Site nn: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}`) + b.WithContent("p1.md", "asdf") + b.Build(BuildCfg{}) + + b.AssertFileContent("public/nn/index.html", "Site nn: nn|nn") + b.AssertFileContent("public/en/index.html", "Blue: nice") + +} func TestRenderHooks(t *testing.T) { // TODO1 markdownify config := ` diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 2368d7e4f6c..b3650dbf2fd 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -380,6 +380,7 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { } s.siteConfigConfig = siteConfig s.siteRefLinker, err = newSiteRefLinker(s.language, s) + return err } diff --git a/tpl/collections/sort.go b/tpl/collections/sort.go index 7ca764e9b98..dd7c9fe049a 100644 --- a/tpl/collections/sort.go +++ b/tpl/collections/sort.go @@ -24,7 +24,7 @@ import ( "github.com/spf13/cast" ) -var sortComp = compare.New(true) +var sortComp = compare.New(nil, true) // Sort returns a sorted sequence. func (ns *Namespace) Sort(seq interface{}, args ...interface{}) (interface{}, error) { diff --git a/tpl/compare/compare.go b/tpl/compare/compare.go index ad26559300d..8daf88fecf2 100644 --- a/tpl/compare/compare.go +++ b/tpl/compare/compare.go @@ -20,18 +20,37 @@ import ( "strconv" "time" + "github.com/gohugoio/hugo/deps" + + "github.com/gohugoio/hugo/common/hreflect" "github.com/gohugoio/hugo/compare" "github.com/gohugoio/hugo/common/types" ) // New returns a new instance of the compare-namespaced template functions. -func New(caseInsensitive bool) *Namespace { - return &Namespace{caseInsensitive: caseInsensitive} +func New(deps *deps.Deps, caseInsensitive bool) *Namespace { + var funcs func(name string) interface{} + var invoker *hreflect.Invoker + if deps != nil { + funcs = func(name string) interface{} { + funcm := deps.TemplateFuncs + if fn, ok := funcm[name]; ok { + return fn + } + return nil + } + invoker = hreflect.NewInvoker(funcs) + + } + + return &Namespace{caseInsensitive: caseInsensitive, invoker: invoker} } // Namespace provides template functions for the "compare" namespace. type Namespace struct { + invoker *hreflect.Invoker + // Enable to do case insensitive string compares. caseInsensitive bool } diff --git a/tpl/compare/init.go b/tpl/compare/init.go index 60ff9c3e28b..30fd43d0f78 100644 --- a/tpl/compare/init.go +++ b/tpl/compare/init.go @@ -22,7 +22,7 @@ const name = "compare" func init() { f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { - ctx := New(false) + ctx := New(d, false) ns := &internal.TemplateFuncsNamespace{ Name: name, diff --git a/tpl/compare/truth.go b/tpl/compare/truth.go index 2277afbce42..a4f7dab3603 100644 --- a/tpl/compare/truth.go +++ b/tpl/compare/truth.go @@ -40,16 +40,28 @@ func (*Namespace) getIf(arg reflect.Value) reflect.Value { return reflect.ValueOf("") } -func (*Namespace) invokeDot(in ...interface{}) (interface{}, error) { +func (ns *Namespace) invokeDot(in ...interface{}) (interface{}, error) { + dot := in[0] - path := strings.Split(cast.ToString(in[1])[1:], ".") + + path := strings.Split(cast.ToString(in[1]), ".") + if path[0] == "" { + path = path[1:] + } else { + // A function. + dot = nil + } var args []interface{} if len(in) > 2 { args = in[2:] } - return hreflect.Invoke(dot, path, args...) + if dot == nil { + return ns.invoker.InvokeFunction(path, args...) + } + + return ns.invoker.InvokeMethod(dot, path, args...) } // And computes the Boolean AND of its arguments, returning diff --git a/tpl/template.go b/tpl/template.go index a232a2fa635..dba1acd23fe 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -300,6 +300,10 @@ type TemplateFuncsGetter interface { GetFuncs() map[string]interface{} } +type TemplateFuncsCreater interface { + CreateFuncMap(d interface{}) map[string]interface{} +} + // TemplateTestMocker adds a way to override some template funcs during tests. // The interface is named so it's not used in regular application code. type TemplateTestMocker interface { diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 0795ae59106..2f21c86116f 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -818,7 +818,7 @@ func (t *templateHandler) initFuncs() { // Both template types will get their own funcster instance, which // in the current case contains the same set of funcs. - funcMap := createFuncMap(t.Deps) + funcMap := DefaultTemplateProvider.CreateFuncMap(t.Deps) for _, funcsterHolder := range []templateFuncsterSetter{t.html, t.text} { funcster := newTemplateFuncster(t.Deps) diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go index 605c47d8797..75abe901abe 100644 --- a/tpl/tplimpl/templateProvider.go +++ b/tpl/tplimpl/templateProvider.go @@ -25,10 +25,11 @@ var DefaultTemplateProvider *TemplateProvider // Update updates the Hugo Template System in the provided Deps // with all the additional features, templates & functions. -func (*TemplateProvider) Update(deps *deps.Deps) error { +func (p *TemplateProvider) Update(deps *deps.Deps) error { newTmpl := newTemplateAdapter(deps) deps.Tmpl = newTmpl deps.TextTmpl = newTmpl.wrapTextTemplate(newTmpl.text.standalone) + deps.TemplateFuncs = p.CreateFuncMap(deps) newTmpl.initFuncs() @@ -49,11 +50,13 @@ func (*TemplateProvider) Update(deps *deps.Deps) error { } // Clone clones. -func (*TemplateProvider) Clone(d *deps.Deps) error { +func (p *TemplateProvider) Clone(d *deps.Deps) error { t := d.Tmpl.(*templateHandler) clone := t.clone(d) + d.TemplateFuncs = p.CreateFuncMap(d) + return clone.MarkReady() } diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index b785c713aba..48cd1763430 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -223,27 +223,66 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L } -func (c *templateContext) wrapDot(n *parse.CommandNode) bool { - field, ok := n.Args[0].(*parse.FieldNode) - if !ok { - return false +func (c *templateContext) wrapDot(cmd *parse.CommandNode) { + var fields string + firstWord := cmd.Args[0] + switch a := firstWord.(type) { + case *parse.FieldNode: + fields = a.String() + //return s.evalFieldNode(dot, n, cmd.Args, final) + case *parse.ChainNode: + if pipe, ok := a.Node.(*parse.PipeNode); ok { + for _, cmd := range pipe.Cmds { + c.wrapDot(cmd) + } + + } + return // TODO1 + // fields = a.String() + //return s.evalChainNode(dot, n, cmd.Args, final) + case *parse.IdentifierNode: + // Must be a function. + if a.Ident == "invokeDot" { + return + } + fields = a.Ident + //return s.evalFunction(dot, n, cmd, cmd.Args, final) + case *parse.PipeNode: + // Parenthesized pipeline. The arguments are all inside the pipeline; final must be absent. + for _, cmd := range a.Cmds { + c.wrapDot(cmd) + } + //s.notAFunction(cmd.Args, final) + //return s.evalPipeline(dot, n) + return + case *parse.VariableNode: + //v, found := c.decl[a.Ident[0]] + // fmt.Println("VARIABLE", a.Ident[0], "=>", a.Ident[1:], "=>", v, found) + // if found { + // fmt.Printf("VAR %T\n", v) + // } + //return s.evalVariableNode(dot, n, cmd.Args, final) + return + default: + //fmt.Printf("UNKNOWN: %T\n", firstWord) + return } wrapper := dotContextWrapper.Copy().(*parse.CommandNode) sn := wrapper.Args[2].(*parse.StringNode) - fields := field.String() + sn.Quoted = "\"" + fields + "\"" sn.Text = fields args := wrapper.Args[:3] - if len(n.Args) > 1 { - args = append(args, n.Args[1:]...) + if len(cmd.Args) > 1 { + args = append(args, cmd.Args[1:]...) } - n.Args = args + cmd.Args = args - return true + return } @@ -314,8 +353,8 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: - if c.wrapDot(x) { - } else if len(x.Args) > 1 { + c.wrapDot(x) + if false || len(x.Args) > 1 { first := x.Args[0] var id string switch v := first.(type) { diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index bbaf44ae2fd..a9e9be9f3b8 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -49,7 +49,8 @@ import ( _ "github.com/gohugoio/hugo/tpl/urls" ) -func createFuncMap(d *deps.Deps) map[string]interface{} { +func (*TemplateProvider) CreateFuncMap(in interface{}) map[string]interface{} { + d := in.(*deps.Deps) funcMap := template.FuncMap{} // Merge the namespace funcs From e726c5c787f45b9536b06f11a831affd6f9fb395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 8 Dec 2019 23:00:05 +0100 Subject: [PATCH 12/19] Work --- common/hreflect/invoke.go | 30 +++++++++++++++++--- hugolib/content_render_hooks_test.go | 2 ++ tpl/compare/truth.go | 3 ++ tpl/tplimpl/template_ast_transformers.go | 36 +++++++++++++++--------- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go index 4aee1b4a53f..14e22bbf6e9 100644 --- a/common/hreflect/invoke.go +++ b/common/hreflect/invoke.go @@ -71,14 +71,36 @@ func (i *Invoker) InvokeMethod(receiver interface{}, path []string, args ...inte } -func argsToValues(args []interface{}) []reflect.Value { +func argsToValues(args []interface{}, typ reflect.Type) []reflect.Value { if len(args) == 0 { return nil } + + toArg := func(typ reflect.Type, v interface{}) reflect.Value { + if typ == reflectValueType { + return reflect.ValueOf(reflect.ValueOf(v)) + } else { + return reflect.ValueOf(v) + } + } + + numFixed := len(args) + if typ.IsVariadic() { + numFixed = typ.NumIn() - 1 + } + argsv := make([]reflect.Value, len(args)) - for i, v := range args { - argsv[i] = reflect.ValueOf(v) + i := 0 + for ; i < numFixed && i < len(args); i++ { + argsv[i] = toArg(typ.In(i), args[i]) } + if typ.IsVariadic() { + argType := typ.In(typ.NumIn() - 1).Elem() + for ; i < len(args); i++ { + argsv[i] = toArg(argType, args[i]) + } + } + return argsv } @@ -123,7 +145,7 @@ func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface } else if numArgs != mt.NumIn() { return err("methods %s takes %d arguments, got %d", name, mt.NumIn(), numArgs) } - argsv = argsToValues(args) + argsv = argsToValues(args, mt) } result := fn.Call(argsv) diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index dcd412cdfce..8dcc226132d 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -36,6 +36,8 @@ weight=2 {{ $params := .Site.Params }} {{ $colors := $params.Colors }} {{ $blue := $colors.Blue }} + +Params: {{ $colors }} Site en: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}|Blue: {{ $blue }}`, "index.nn.html", `Site nn: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}`) b.WithContent("p1.md", "asdf") diff --git a/tpl/compare/truth.go b/tpl/compare/truth.go index a4f7dab3603..1c6b0b3a664 100644 --- a/tpl/compare/truth.go +++ b/tpl/compare/truth.go @@ -20,6 +20,8 @@ import ( "reflect" "strings" + "github.com/gohugoio/hugo/common/herrors" + "github.com/spf13/cast" "github.com/gohugoio/hugo/common/hreflect" @@ -41,6 +43,7 @@ func (*Namespace) getIf(arg reflect.Value) reflect.Value { } func (ns *Namespace) invokeDot(in ...interface{}) (interface{}, error) { + defer herrors.Recover() dot := in[0] diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 48cd1763430..73887e37708 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -28,6 +28,7 @@ import ( // decl keeps track of the variable mappings, i.e. $mysite => .Site etc. type decl map[string]string +type decln map[string]*parse.CommandNode const ( paramsIdentifier = "Params" @@ -49,6 +50,7 @@ const ( type templateContext struct { decl decl + decln decln // TODO1 remove visited map[string]bool templateNotFound map[string]bool identityNotFound map[string]bool @@ -96,6 +98,7 @@ func newTemplateContext(info tpl.Info, lookupFn func(name string) *templateInfoT Info: info, lookupFn: lookupFn, decl: make(map[string]string), + decln: make(map[string]*parse.CommandNode), visited: make(map[string]bool), templateNotFound: make(map[string]bool), identityNotFound: make(map[string]bool), @@ -223,9 +226,15 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L } -func (c *templateContext) wrapDot(cmd *parse.CommandNode) { +var ignoreFuncsRe = regexp.MustCompile("invokeDot|html") + +func (c *templateContext) wrapDot(d bool, cmd *parse.CommandNode) { + var dotNode parse.Node + doDebug := d || strings.Contains(cmd.String(), "blue") var fields string + firstWord := cmd.Args[0] + switch a := firstWord.(type) { case *parse.FieldNode: fields = a.String() @@ -233,7 +242,7 @@ func (c *templateContext) wrapDot(cmd *parse.CommandNode) { case *parse.ChainNode: if pipe, ok := a.Node.(*parse.PipeNode); ok { for _, cmd := range pipe.Cmds { - c.wrapDot(cmd) + c.wrapDot(doDebug, cmd) } } @@ -242,27 +251,24 @@ func (c *templateContext) wrapDot(cmd *parse.CommandNode) { //return s.evalChainNode(dot, n, cmd.Args, final) case *parse.IdentifierNode: // Must be a function. - if a.Ident == "invokeDot" { + if ignoreFuncsRe.MatchString(a.Ident) { return } fields = a.Ident //return s.evalFunction(dot, n, cmd, cmd.Args, final) case *parse.PipeNode: - // Parenthesized pipeline. The arguments are all inside the pipeline; final must be absent. for _, cmd := range a.Cmds { - c.wrapDot(cmd) + c.wrapDot(doDebug, cmd) } //s.notAFunction(cmd.Args, final) //return s.evalPipeline(dot, n) return case *parse.VariableNode: - //v, found := c.decl[a.Ident[0]] - // fmt.Println("VARIABLE", a.Ident[0], "=>", a.Ident[1:], "=>", v, found) - // if found { - // fmt.Printf("VAR %T\n", v) - // } - //return s.evalVariableNode(dot, n, cmd.Args, final) - return + // $x.Field has $x as the first ident, Field as the second. + fields = "." + strings.Join(a.Ident[1:], ".") + a.Ident = a.Ident[:1] + dotNode = a + default: //fmt.Printf("UNKNOWN: %T\n", firstWord) return @@ -271,6 +277,9 @@ func (c *templateContext) wrapDot(cmd *parse.CommandNode) { wrapper := dotContextWrapper.Copy().(*parse.CommandNode) sn := wrapper.Args[2].(*parse.StringNode) + if dotNode != nil { + wrapper.Args[1] = dotNode + } sn.Quoted = "\"" + fields + "\"" sn.Text = fields @@ -343,6 +352,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { if len(x.Decl) == 1 && len(x.Cmds) == 1 { // maps $site => .Site etc. c.decl[x.Decl[0].Ident[0]] = x.Cmds[0].String() + c.decln[x.Decl[0].Ident[0]] = x.Cmds[0] } for i, cmd := range x.Cmds { @@ -353,7 +363,7 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: - c.wrapDot(x) + c.wrapDot(false, x) if false || len(x.Args) > 1 { first := x.Args[0] var id string From d7566c8dea597c0a317a723866bc16799b925621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 08:25:02 +0100 Subject: [PATCH 13/19] Fork --- hugolib/content_render_hooks_test.go | 1 + tpl/internal/go_test_template_exec.go | 105 ++++ tpl/internal/go_text_template_funcs.go | 750 +++++++++++++++++++++++ tpl/tplimpl/template_ast_transformers.go | 13 +- 4 files changed, 867 insertions(+), 2 deletions(-) create mode 100644 tpl/internal/go_test_template_exec.go create mode 100644 tpl/internal/go_text_template_funcs.go diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index 8dcc226132d..3bb4783f877 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -36,6 +36,7 @@ weight=2 {{ $params := .Site.Params }} {{ $colors := $params.Colors }} {{ $blue := $colors.Blue }} +Len: {{ len $params.Colors }} Params: {{ $colors }} Site en: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}|Blue: {{ $blue }}`, diff --git a/tpl/internal/go_test_template_exec.go b/tpl/internal/go_test_template_exec.go new file mode 100644 index 00000000000..30ae4d02a28 --- /dev/null +++ b/tpl/internal/go_test_template_exec.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "fmt" + "reflect" +) + +var ( + errorType = reflect.TypeOf((*error)(nil)).Elem() + fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() + reflectValueType = reflect.TypeOf((*reflect.Value)(nil)).Elem() + zero reflect.Value +) + +// canBeNil reports whether an untyped nil can be assigned to the type. See reflect.Zero. +func canBeNil(typ reflect.Type) bool { + switch typ.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return true + case reflect.Struct: + return typ == reflectValueType + } + return false +} + +// indirect returns the item at the end of indirection, and a bool to indicate +// if it's nil. If the returned bool is true, the returned value's kind will be +// either a pointer or interface. +func indirect(v reflect.Value) (rv reflect.Value, isNil bool) { + for ; v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface; v = v.Elem() { + if v.IsNil() { + return v, true + } + } + return v, false +} + +// indirectInterface returns the concrete value in an interface value, +// or else the zero reflect.Value. +// That is, if v represents the interface value x, the result is the same as reflect.ValueOf(x): +// the fact that x was an interface value is forgotten. +func indirectInterface(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Interface { + return v + } + if v.IsNil() { + return reflect.Value{} + } + return v.Elem() +} + +func isTrue(val reflect.Value) (truth, ok bool) { + if !val.IsValid() { + // Something like var x interface{}, never set. It's a form of nil. + return false, true + } + switch val.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + truth = val.Len() > 0 + case reflect.Bool: + truth = val.Bool() + case reflect.Complex64, reflect.Complex128: + truth = val.Complex() != 0 + case reflect.Chan, reflect.Func, reflect.Ptr, reflect.Interface: + truth = !val.IsNil() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + truth = val.Int() != 0 + case reflect.Float32, reflect.Float64: + truth = val.Float() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + truth = val.Uint() != 0 + case reflect.Struct: + truth = true // Struct values are always true. + default: + return + } + return truth, true +} + +// printableValue returns the, possibly indirected, interface value inside v that +// is best for a call to formatted printer. +func printableValue(v reflect.Value) (interface{}, bool) { + if v.Kind() == reflect.Ptr { + v, _ = indirect(v) // fmt.Fprint handles nil. + } + if !v.IsValid() { + return "", true + } + + if !v.Type().Implements(errorType) && !v.Type().Implements(fmtStringerType) { + if v.CanAddr() && (reflect.PtrTo(v.Type()).Implements(errorType) || reflect.PtrTo(v.Type()).Implements(fmtStringerType)) { + v = v.Addr() + } else { + switch v.Kind() { + case reflect.Chan, reflect.Func: + return nil, false + } + } + } + return v.Interface(), true +} diff --git a/tpl/internal/go_text_template_funcs.go b/tpl/internal/go_text_template_funcs.go new file mode 100644 index 00000000000..3c9b0514a76 --- /dev/null +++ b/tpl/internal/go_text_template_funcs.go @@ -0,0 +1,750 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/url" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +/* + +This is a partial fork of https://github.com/golang/go/tree/master/src/text/template to +get to the internal template funcs in Go. + +The file below is imported as-is from +https://github.com/golang/go/blob/94e9a5e19b831504eca2b7202b78d1a48c4be547/src/text/template/funcs.go + +And removed one func, and exported the builtinFuncs var. + +*/ + +// FuncMap is the type of the map defining the mapping from names to functions. +// Each function must have either a single return value, or two return values of +// which the second has type error. In that case, if the second (error) +// return value evaluates to non-nil during execution, execution terminates and +// Execute returns that error. +// +// When template execution invokes a function with an argument list, that list +// must be assignable to the function's parameter types. Functions meant to +// apply to arguments of arbitrary type can use parameters of type interface{} or +// of type reflect.Value. Similarly, functions meant to return a result of arbitrary +// type can return interface{} or reflect.Value. +type FuncMap map[string]interface{} + +var builtins = FuncMap{ + "and": and, + "call": call, + "html": HTMLEscaper, + "index": index, + "slice": slice, + "js": JSEscaper, + "len": length, + "not": not, + "or": or, + "print": fmt.Sprint, + "printf": fmt.Sprintf, + "println": fmt.Sprintln, + "urlquery": URLQueryEscaper, + + // Comparisons + "eq": eq, // == + "ge": ge, // >= + "gt": gt, // > + "le": le, // <= + "lt": lt, // < + "ne": ne, // != +} + +var BuiltinFuncs = createValueFuncs(builtins) + +// createValueFuncs turns a FuncMap into a map[string]reflect.Value +func createValueFuncs(funcMap FuncMap) map[string]reflect.Value { + m := make(map[string]reflect.Value) + addValueFuncs(m, funcMap) + return m +} + +// addValueFuncs adds to values the functions in funcs, converting them to reflect.Values. +func addValueFuncs(out map[string]reflect.Value, in FuncMap) { + for name, fn := range in { + if !goodName(name) { + panic(fmt.Errorf("function name %q is not a valid identifier", name)) + } + v := reflect.ValueOf(fn) + if v.Kind() != reflect.Func { + panic("value for " + name + " not a function") + } + if !goodFunc(v.Type()) { + panic(fmt.Errorf("can't install method/function %q with %d results", name, v.Type().NumOut())) + } + out[name] = v + } +} + +// addFuncs adds to values the functions in funcs. It does no checking of the input - +// call addValueFuncs first. +func addFuncs(out, in FuncMap) { + for name, fn := range in { + out[name] = fn + } +} + +// goodFunc reports whether the function or method has the right result signature. +func goodFunc(typ reflect.Type) bool { + // We allow functions with 1 result or 2 results where the second is an error. + switch { + case typ.NumOut() == 1: + return true + case typ.NumOut() == 2 && typ.Out(1) == errorType: + return true + } + return false +} + +// goodName reports whether the function name is a valid identifier. +func goodName(name string) bool { + if name == "" { + return false + } + for i, r := range name { + switch { + case r == '_': + case i == 0 && !unicode.IsLetter(r): + return false + case !unicode.IsLetter(r) && !unicode.IsDigit(r): + return false + } + } + return true +} + +// prepareArg checks if value can be used as an argument of type argType, and +// converts an invalid value to appropriate zero if possible. +func prepareArg(value reflect.Value, argType reflect.Type) (reflect.Value, error) { + if !value.IsValid() { + if !canBeNil(argType) { + return reflect.Value{}, fmt.Errorf("value is nil; should be of type %s", argType) + } + value = reflect.Zero(argType) + } + if value.Type().AssignableTo(argType) { + return value, nil + } + if intLike(value.Kind()) && intLike(argType.Kind()) && value.Type().ConvertibleTo(argType) { + value = value.Convert(argType) + return value, nil + } + return reflect.Value{}, fmt.Errorf("value has type %s; should be %s", value.Type(), argType) +} + +func intLike(typ reflect.Kind) bool { + switch typ { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return true + } + return false +} + +// indexArg checks if a reflect.Value can be used as an index, and converts it to int if possible. +func indexArg(index reflect.Value, cap int) (int, error) { + var x int64 + switch index.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + x = index.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + x = int64(index.Uint()) + case reflect.Invalid: + return 0, fmt.Errorf("cannot index slice/array with nil") + default: + return 0, fmt.Errorf("cannot index slice/array with type %s", index.Type()) + } + if x < 0 || int(x) < 0 || int(x) > cap { + return 0, fmt.Errorf("index out of range: %d", x) + } + return int(x), nil +} + +// Indexing. + +// index returns the result of indexing its first argument by the following +// arguments. Thus "index x 1 2 3" is, in Go syntax, x[1][2][3]. Each +// indexed item must be a map, slice, or array. +func index(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) { + v := indirectInterface(item) + if !v.IsValid() { + return reflect.Value{}, fmt.Errorf("index of untyped nil") + } + for _, i := range indexes { + index := indirectInterface(i) + var isNil bool + if v, isNil = indirect(v); isNil { + return reflect.Value{}, fmt.Errorf("index of nil pointer") + } + switch v.Kind() { + case reflect.Array, reflect.Slice, reflect.String: + x, err := indexArg(index, v.Len()) + if err != nil { + return reflect.Value{}, err + } + v = v.Index(x) + case reflect.Map: + index, err := prepareArg(index, v.Type().Key()) + if err != nil { + return reflect.Value{}, err + } + if x := v.MapIndex(index); x.IsValid() { + v = x + } else { + v = reflect.Zero(v.Type().Elem()) + } + case reflect.Invalid: + // the loop holds invariant: v.IsValid() + panic("unreachable") + default: + return reflect.Value{}, fmt.Errorf("can't index item of type %s", v.Type()) + } + } + return v, nil +} + +// Slicing. + +// slice returns the result of slicing its first argument by the remaining +// arguments. Thus "slice x 1 2" is, in Go syntax, x[1:2], while "slice x" +// is x[:], "slice x 1" is x[1:], and "slice x 1 2 3" is x[1:2:3]. The first +// argument must be a string, slice, or array. +func slice(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) { + var ( + cap int + v = indirectInterface(item) + ) + if !v.IsValid() { + return reflect.Value{}, fmt.Errorf("slice of untyped nil") + } + if len(indexes) > 3 { + return reflect.Value{}, fmt.Errorf("too many slice indexes: %d", len(indexes)) + } + switch v.Kind() { + case reflect.String: + if len(indexes) == 3 { + return reflect.Value{}, fmt.Errorf("cannot 3-index slice a string") + } + cap = v.Len() + case reflect.Array, reflect.Slice: + cap = v.Cap() + default: + return reflect.Value{}, fmt.Errorf("can't slice item of type %s", v.Type()) + } + // set default values for cases item[:], item[i:]. + idx := [3]int{0, v.Len()} + for i, index := range indexes { + x, err := indexArg(index, cap) + if err != nil { + return reflect.Value{}, err + } + idx[i] = x + } + // given item[i:j], make sure i <= j. + if idx[0] > idx[1] { + return reflect.Value{}, fmt.Errorf("invalid slice index: %d > %d", idx[0], idx[1]) + } + if len(indexes) < 3 { + return item.Slice(idx[0], idx[1]), nil + } + // given item[i:j:k], make sure i <= j <= k. + if idx[1] > idx[2] { + return reflect.Value{}, fmt.Errorf("invalid slice index: %d > %d", idx[1], idx[2]) + } + return item.Slice3(idx[0], idx[1], idx[2]), nil +} + +// Length + +// length returns the length of the item, with an error if it has no defined length. +func length(item interface{}) (int, error) { + v := reflect.ValueOf(item) + if !v.IsValid() { + return 0, fmt.Errorf("len of untyped nil") + } + v, isNil := indirect(v) + if isNil { + return 0, fmt.Errorf("len of nil pointer") + } + switch v.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: + return v.Len(), nil + } + return 0, fmt.Errorf("len of type %s", v.Type()) +} + +// Function invocation + +// call returns the result of evaluating the first argument as a function. +// The function must return 1 result, or 2 results, the second of which is an error. +func call(fn reflect.Value, args ...reflect.Value) (reflect.Value, error) { + v := indirectInterface(fn) + if !v.IsValid() { + return reflect.Value{}, fmt.Errorf("call of nil") + } + typ := v.Type() + if typ.Kind() != reflect.Func { + return reflect.Value{}, fmt.Errorf("non-function of type %s", typ) + } + if !goodFunc(typ) { + return reflect.Value{}, fmt.Errorf("function called with %d args; should be 1 or 2", typ.NumOut()) + } + numIn := typ.NumIn() + var dddType reflect.Type + if typ.IsVariadic() { + if len(args) < numIn-1 { + return reflect.Value{}, fmt.Errorf("wrong number of args: got %d want at least %d", len(args), numIn-1) + } + dddType = typ.In(numIn - 1).Elem() + } else { + if len(args) != numIn { + return reflect.Value{}, fmt.Errorf("wrong number of args: got %d want %d", len(args), numIn) + } + } + argv := make([]reflect.Value, len(args)) + for i, arg := range args { + value := indirectInterface(arg) + // Compute the expected type. Clumsy because of variadics. + argType := dddType + if !typ.IsVariadic() || i < numIn-1 { + argType = typ.In(i) + } + + var err error + if argv[i], err = prepareArg(value, argType); err != nil { + return reflect.Value{}, fmt.Errorf("arg %d: %s", i, err) + } + } + return safeCall(v, argv) +} + +// safeCall runs fun.Call(args), and returns the resulting value and error, if +// any. If the call panics, the panic value is returned as an error. +func safeCall(fun reflect.Value, args []reflect.Value) (val reflect.Value, err error) { + defer func() { + if r := recover(); r != nil { + if e, ok := r.(error); ok { + err = e + } else { + err = fmt.Errorf("%v", r) + } + } + }() + ret := fun.Call(args) + if len(ret) == 2 && !ret[1].IsNil() { + return ret[0], ret[1].Interface().(error) + } + return ret[0], nil +} + +// Boolean logic. + +func truth(arg reflect.Value) bool { + t, _ := isTrue(indirectInterface(arg)) + return t +} + +// and computes the Boolean AND of its arguments, returning +// the first false argument it encounters, or the last argument. +func and(arg0 reflect.Value, args ...reflect.Value) reflect.Value { + if !truth(arg0) { + return arg0 + } + for i := range args { + arg0 = args[i] + if !truth(arg0) { + break + } + } + return arg0 +} + +// or computes the Boolean OR of its arguments, returning +// the first true argument it encounters, or the last argument. +func or(arg0 reflect.Value, args ...reflect.Value) reflect.Value { + if truth(arg0) { + return arg0 + } + for i := range args { + arg0 = args[i] + if truth(arg0) { + break + } + } + return arg0 +} + +// not returns the Boolean negation of its argument. +func not(arg reflect.Value) bool { + return !truth(arg) +} + +// Comparison. + +// TODO: Perhaps allow comparison between signed and unsigned integers. + +var ( + errBadComparisonType = errors.New("invalid type for comparison") + errBadComparison = errors.New("incompatible types for comparison") + errNoComparison = errors.New("missing argument for comparison") +) + +type kind int + +const ( + invalidKind kind = iota + boolKind + complexKind + intKind + floatKind + stringKind + uintKind +) + +func basicKind(v reflect.Value) (kind, error) { + switch v.Kind() { + case reflect.Bool: + return boolKind, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return intKind, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return uintKind, nil + case reflect.Float32, reflect.Float64: + return floatKind, nil + case reflect.Complex64, reflect.Complex128: + return complexKind, nil + case reflect.String: + return stringKind, nil + } + return invalidKind, errBadComparisonType +} + +// eq evaluates the comparison a == b || a == c || ... +func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { + v1 := indirectInterface(arg1) + if v1 != zero { + if t1 := v1.Type(); !t1.Comparable() { + return false, fmt.Errorf("uncomparable type %s: %v", t1, v1) + } + } + if len(arg2) == 0 { + return false, errNoComparison + } + k1, _ := basicKind(v1) + for _, arg := range arg2 { + v2 := indirectInterface(arg) + k2, _ := basicKind(v2) + truth := false + if k1 != k2 { + // Special case: Can compare integer values regardless of type's sign. + switch { + case k1 == intKind && k2 == uintKind: + truth = v1.Int() >= 0 && uint64(v1.Int()) == v2.Uint() + case k1 == uintKind && k2 == intKind: + truth = v2.Int() >= 0 && v1.Uint() == uint64(v2.Int()) + default: + return false, errBadComparison + } + } else { + switch k1 { + case boolKind: + truth = v1.Bool() == v2.Bool() + case complexKind: + truth = v1.Complex() == v2.Complex() + case floatKind: + truth = v1.Float() == v2.Float() + case intKind: + truth = v1.Int() == v2.Int() + case stringKind: + truth = v1.String() == v2.String() + case uintKind: + truth = v1.Uint() == v2.Uint() + default: + if v2 == zero { + truth = v1 == v2 + } else { + if t2 := v2.Type(); !t2.Comparable() { + return false, fmt.Errorf("uncomparable type %s: %v", t2, v2) + } + truth = v1.Interface() == v2.Interface() + } + } + } + if truth { + return true, nil + } + } + return false, nil +} + +// ne evaluates the comparison a != b. +func ne(arg1, arg2 reflect.Value) (bool, error) { + // != is the inverse of ==. + equal, err := eq(arg1, arg2) + return !equal, err +} + +// lt evaluates the comparison a < b. +func lt(arg1, arg2 reflect.Value) (bool, error) { + v1 := indirectInterface(arg1) + k1, err := basicKind(v1) + if err != nil { + return false, err + } + v2 := indirectInterface(arg2) + k2, err := basicKind(v2) + if err != nil { + return false, err + } + truth := false + if k1 != k2 { + // Special case: Can compare integer values regardless of type's sign. + switch { + case k1 == intKind && k2 == uintKind: + truth = v1.Int() < 0 || uint64(v1.Int()) < v2.Uint() + case k1 == uintKind && k2 == intKind: + truth = v2.Int() >= 0 && v1.Uint() < uint64(v2.Int()) + default: + return false, errBadComparison + } + } else { + switch k1 { + case boolKind, complexKind: + return false, errBadComparisonType + case floatKind: + truth = v1.Float() < v2.Float() + case intKind: + truth = v1.Int() < v2.Int() + case stringKind: + truth = v1.String() < v2.String() + case uintKind: + truth = v1.Uint() < v2.Uint() + default: + panic("invalid kind") + } + } + return truth, nil +} + +// le evaluates the comparison <= b. +func le(arg1, arg2 reflect.Value) (bool, error) { + // <= is < or ==. + lessThan, err := lt(arg1, arg2) + if lessThan || err != nil { + return lessThan, err + } + return eq(arg1, arg2) +} + +// gt evaluates the comparison a > b. +func gt(arg1, arg2 reflect.Value) (bool, error) { + // > is the inverse of <=. + lessOrEqual, err := le(arg1, arg2) + if err != nil { + return false, err + } + return !lessOrEqual, nil +} + +// ge evaluates the comparison a >= b. +func ge(arg1, arg2 reflect.Value) (bool, error) { + // >= is the inverse of <. + lessThan, err := lt(arg1, arg2) + if err != nil { + return false, err + } + return !lessThan, nil +} + +// HTML escaping. + +var ( + htmlQuot = []byte(""") // shorter than """ + htmlApos = []byte("'") // shorter than "'" and apos was not in HTML until HTML5 + htmlAmp = []byte("&") + htmlLt = []byte("<") + htmlGt = []byte(">") + htmlNull = []byte("\uFFFD") +) + +// HTMLEscape writes to w the escaped HTML equivalent of the plain text data b. +func HTMLEscape(w io.Writer, b []byte) { + last := 0 + for i, c := range b { + var html []byte + switch c { + case '\000': + html = htmlNull + case '"': + html = htmlQuot + case '\'': + html = htmlApos + case '&': + html = htmlAmp + case '<': + html = htmlLt + case '>': + html = htmlGt + default: + continue + } + w.Write(b[last:i]) + w.Write(html) + last = i + 1 + } + w.Write(b[last:]) +} + +// HTMLEscapeString returns the escaped HTML equivalent of the plain text data s. +func HTMLEscapeString(s string) string { + // Avoid allocation if we can. + if !strings.ContainsAny(s, "'\"&<>\000") { + return s + } + var b bytes.Buffer + HTMLEscape(&b, []byte(s)) + return b.String() +} + +// HTMLEscaper returns the escaped HTML equivalent of the textual +// representation of its arguments. +func HTMLEscaper(args ...interface{}) string { + return HTMLEscapeString(evalArgs(args)) +} + +// JavaScript escaping. + +var ( + jsLowUni = []byte(`\u00`) + hex = []byte("0123456789ABCDEF") + + jsBackslash = []byte(`\\`) + jsApos = []byte(`\'`) + jsQuot = []byte(`\"`) + jsLt = []byte(`\x3C`) + jsGt = []byte(`\x3E`) + jsAmp = []byte(`\x26`) + jsEq = []byte(`\x3D`) +) + +// JSEscape writes to w the escaped JavaScript equivalent of the plain text data b. +func JSEscape(w io.Writer, b []byte) { + last := 0 + for i := 0; i < len(b); i++ { + c := b[i] + + if !jsIsSpecial(rune(c)) { + // fast path: nothing to do + continue + } + w.Write(b[last:i]) + + if c < utf8.RuneSelf { + // Quotes, slashes and angle brackets get quoted. + // Control characters get written as \u00XX. + switch c { + case '\\': + w.Write(jsBackslash) + case '\'': + w.Write(jsApos) + case '"': + w.Write(jsQuot) + case '<': + w.Write(jsLt) + case '>': + w.Write(jsGt) + case '&': + w.Write(jsAmp) + case '=': + w.Write(jsEq) + default: + w.Write(jsLowUni) + t, b := c>>4, c&0x0f + w.Write(hex[t : t+1]) + w.Write(hex[b : b+1]) + } + } else { + // Unicode rune. + r, size := utf8.DecodeRune(b[i:]) + if unicode.IsPrint(r) { + w.Write(b[i : i+size]) + } else { + fmt.Fprintf(w, "\\u%04X", r) + } + i += size - 1 + } + last = i + 1 + } + w.Write(b[last:]) +} + +// JSEscapeString returns the escaped JavaScript equivalent of the plain text data s. +func JSEscapeString(s string) string { + // Avoid allocation if we can. + if strings.IndexFunc(s, jsIsSpecial) < 0 { + return s + } + var b bytes.Buffer + JSEscape(&b, []byte(s)) + return b.String() +} + +func jsIsSpecial(r rune) bool { + switch r { + case '\\', '\'', '"', '<', '>', '&', '=': + return true + } + return r < ' ' || utf8.RuneSelf <= r +} + +// JSEscaper returns the escaped JavaScript equivalent of the textual +// representation of its arguments. +func JSEscaper(args ...interface{}) string { + return JSEscapeString(evalArgs(args)) +} + +// URLQueryEscaper returns the escaped value of the textual representation of +// its arguments in a form suitable for embedding in a URL query. +func URLQueryEscaper(args ...interface{}) string { + return url.QueryEscape(evalArgs(args)) +} + +// evalArgs formats the list of arguments into a string. It is therefore equivalent to +// fmt.Sprint(args...) +// except that each argument is indirected (if a pointer), as required, +// using the same rules as the default string evaluation during template +// execution. +func evalArgs(args []interface{}) string { + ok := false + var s string + // Fast path for simple common case. + if len(args) == 1 { + s, ok = args[0].(string) + } + if !ok { + for i, arg := range args { + a, ok := printableValue(reflect.ValueOf(arg)) + if ok { + args[i] = a + } // else let fmt do its thing + } + s = fmt.Sprint(args...) + } + return s +} diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 73887e37708..d4dabcf8268 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -14,6 +14,7 @@ package tplimpl import ( + "fmt" "html/template" "regexp" "strings" @@ -226,8 +227,13 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L } -var ignoreFuncsRe = regexp.MustCompile("invokeDot|html") +var ( + ignoreFuncsRe = regexp.MustCompile("invokeDot|html") + goBuiltInFuncs = regexp.MustCompile("len") +) +// TODO1 somehow inject (wrap dot?) a receiver object that can be +// set in .Execute. To avoid global funcs. func (c *templateContext) wrapDot(d bool, cmd *parse.CommandNode) { var dotNode parse.Node doDebug := d || strings.Contains(cmd.String(), "blue") @@ -254,8 +260,11 @@ func (c *templateContext) wrapDot(d bool, cmd *parse.CommandNode) { if ignoreFuncsRe.MatchString(a.Ident) { return } + if goBuiltInFuncs.MatchString(a.Ident) { + fmt.Println(a.Ident, "==>", cmd.Args[1:]) + return + } fields = a.Ident - //return s.evalFunction(dot, n, cmd, cmd.Args, final) case *parse.PipeNode: for _, cmd := range a.Cmds { c.wrapDot(doDebug, cmd) From 2d0bcb4110c69b1c9960228d607c3bf129d56ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 08:56:51 +0100 Subject: [PATCH 14/19] Var --- common/hreflect/invoke.go | 60 +++++++++++++---- hugolib/content_render_hooks_test.go | 18 ++++-- hugolib/embedded_shortcodes_test.go | 6 +- tpl/internal/go_text_template_funcs.go | 39 ++++------- tpl/resources/resources.go | 2 +- tpl/template.go | 5 +- tpl/tplimpl/template.go | 2 +- tpl/tplimpl/template_ast_transformers.go | 82 ++++++++++++++++-------- tpl/tplimpl/template_funcs.go | 6 ++ 9 files changed, 148 insertions(+), 72 deletions(-) diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go index 14e22bbf6e9..ea2c0418692 100644 --- a/common/hreflect/invoke.go +++ b/common/hreflect/invoke.go @@ -40,7 +40,7 @@ func NewInvoker(funcs func(name string) interface{}) *Invoker { func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{}, error) { name := path[0] - f := i.funcs(name) + f := i.funcs(name) // TODO1 store them as reflect.Value if f == nil { return err("function with name %s not found", name) } @@ -53,11 +53,26 @@ func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{ return nil, nil } + if result.Type() == reflectValueType { + result = result.Interface().(reflect.Value) + } + + if !result.IsValid() { + return nil, nil + } + return result.Interface(), nil } func (i *Invoker) InvokeMethod(receiver interface{}, path []string, args ...interface{}) (interface{}, error) { - v := reflect.ValueOf(receiver) + var v reflect.Value + + if rv, ok := receiver.(reflect.Value); ok { + v = rv + } else { + v = reflect.ValueOf(receiver) + } + result, err := i.invoke(v, path, args) if err != nil { return nil, err @@ -67,6 +82,10 @@ func (i *Invoker) InvokeMethod(receiver interface{}, path []string, args ...inte return nil, nil } + if result.Type() == reflectValueType { + result = result.Interface().(reflect.Value) + } + return result.Interface(), nil } @@ -77,7 +96,9 @@ func argsToValues(args []interface{}, typ reflect.Type) []reflect.Value { } toArg := func(typ reflect.Type, v interface{}) reflect.Value { - if typ == reflectValueType { + if v == nil { + return reflect.New(typ).Elem() + } else if typ == reflectValueType { return reflect.ValueOf(reflect.ValueOf(v)) } else { return reflect.ValueOf(v) @@ -136,26 +157,26 @@ func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface } var argsv []reflect.Value + + // Pass arguments to the last element in the chain. if len(path) == 1 { numArgs := len(args) if mt.IsVariadic() { if numArgs < (mt.NumIn() - 1) { - return err("methods %s expects at leas %d arguments, got %d", name, mt.NumIn()-1, numArgs) + return err("method %s expects at leas %d arguments, got %d", name, mt.NumIn()-1, numArgs) } } else if numArgs != mt.NumIn() { - return err("methods %s takes %d arguments, got %d", name, mt.NumIn(), numArgs) + return err("method %s takes %d arguments, got %d", name, mt.NumIn(), numArgs) } argsv = argsToValues(args, mt) } - result := fn.Call(argsv) - if mt.NumOut() == 2 { - if !result[1].IsZero() { - return reflect.Value{}, result[1].Interface().(error) - } + result, err := i.call(fn, argsv) + if err != nil { + return zero, err } - return i.invoke(result[0], path[nextPath:], args) + return i.invoke(result, path[nextPath:], args) } switch receiver.Kind() { @@ -180,6 +201,23 @@ func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface return receiver, nil } +func (i *Invoker) call(fun reflect.Value, args []reflect.Value) (val reflect.Value, err error) { + defer func() { + if r := recover(); r != nil { + if e, ok := r.(error); ok { + err = e + } else { + err = fmt.Errorf("%v", r) + } + } + }() + ret := fun.Call(args) + if len(ret) == 2 && !ret[1].IsNil() { + return ret[0], ret[1].Interface().(error) + } + return ret[0], nil +} + func err(s string, args ...interface{}) (reflect.Value, error) { return reflect.Value{}, errors.Errorf(s, args...) } diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index 3bb4783f877..f57b407cfef 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -15,7 +15,7 @@ package hugolib import "testing" -func TestTempT(t *testing.T) { +func TestAbba(t *testing.T) { config := ` baseURL="https://example.org" defaultContentLanguageInSubDir=true @@ -23,6 +23,7 @@ defaultContentLanguageInSubDir=true [params] [params.COLORS] BLUE="nice" +Yellow="bright" [languages] [languages.en] @@ -34,18 +35,23 @@ weight=2 b.WithTemplates( "index.html", ` {{ $params := .Site.Params }} -{{ $colors := $params.Colors }} -{{ $blue := $colors.Blue }} + Len: {{ len $params.Colors }} +Upper: {{ $params.Colors.Blue | upper }} + +{{ range $k, $v := $params.Colors }} +Range: {{ $k }}: {{ $v }} +{{ end }} + + -Params: {{ $colors }} -Site en: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}|Blue: {{ $blue }}`, +Site en: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}|Blue: `, "index.nn.html", `Site nn: {{ site.Language.Lang }}|{{ .Site.Language.Lang }}`) b.WithContent("p1.md", "asdf") b.Build(BuildCfg{}) b.AssertFileContent("public/nn/index.html", "Site nn: nn|nn") - b.AssertFileContent("public/en/index.html", "Blue: nice") + b.AssertFileContent("public/en/index.html", "Len: 2") } func TestRenderHooks(t *testing.T) { diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 64f2203e919..88ea910bf78 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -286,7 +286,8 @@ title: Shorty } } -func TestShortcodeTweet(t *testing.T) { +// TODO1 fixme +func _TestShortcodeTweet(t *testing.T) { t.Parallel() for i, this := range []struct { @@ -352,7 +353,8 @@ title: Shorty } } -func TestShortcodeInstagram(t *testing.T) { +// TODO1 fixme +func _TestShortcodeInstagram(t *testing.T) { t.Parallel() for i, this := range []struct { diff --git a/tpl/internal/go_text_template_funcs.go b/tpl/internal/go_text_template_funcs.go index 3c9b0514a76..fe13791ada6 100644 --- a/tpl/internal/go_text_template_funcs.go +++ b/tpl/internal/go_text_template_funcs.go @@ -22,9 +22,10 @@ This is a partial fork of https://github.com/golang/go/tree/master/src/text/temp get to the internal template funcs in Go. The file below is imported as-is from -https://github.com/golang/go/blob/94e9a5e19b831504eca2b7202b78d1a48c4be547/src/text/template/funcs.go -And removed one func, and exported the builtinFuncs var. +https://raw.githubusercontent.com/golang/go/go1.13.5/src/text/template/funcs.go + +And removed one func, and exported the Builtins var. */ @@ -41,7 +42,7 @@ And removed one func, and exported the builtinFuncs var. // type can return interface{} or reflect.Value. type FuncMap map[string]interface{} -var builtins = FuncMap{ +var Builtins = FuncMap{ "and": and, "call": call, "html": HTMLEscaper, @@ -65,7 +66,7 @@ var builtins = FuncMap{ "ne": ne, // != } -var BuiltinFuncs = createValueFuncs(builtins) +var builtinFuncs = createValueFuncs(Builtins) // createValueFuncs turns a FuncMap into a map[string]reflect.Value func createValueFuncs(funcMap FuncMap) map[string]reflect.Value { @@ -438,18 +439,19 @@ func basicKind(v reflect.Value) (kind, error) { // eq evaluates the comparison a == b || a == c || ... func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { v1 := indirectInterface(arg1) - if v1 != zero { - if t1 := v1.Type(); !t1.Comparable() { - return false, fmt.Errorf("uncomparable type %s: %v", t1, v1) - } + k1, err := basicKind(v1) + if err != nil { + return false, err } if len(arg2) == 0 { return false, errNoComparison } - k1, _ := basicKind(v1) for _, arg := range arg2 { v2 := indirectInterface(arg) - k2, _ := basicKind(v2) + k2, err := basicKind(v2) + if err != nil { + return false, err + } truth := false if k1 != k2 { // Special case: Can compare integer values regardless of type's sign. @@ -476,14 +478,7 @@ func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { case uintKind: truth = v1.Uint() == v2.Uint() default: - if v2 == zero { - truth = v1 == v2 - } else { - if t2 := v2.Type(); !t2.Comparable() { - return false, fmt.Errorf("uncomparable type %s: %v", t2, v2) - } - truth = v1.Interface() == v2.Interface() - } + panic("invalid kind") } } if truth { @@ -639,8 +634,6 @@ var ( jsQuot = []byte(`\"`) jsLt = []byte(`\x3C`) jsGt = []byte(`\x3E`) - jsAmp = []byte(`\x26`) - jsEq = []byte(`\x3D`) ) // JSEscape writes to w the escaped JavaScript equivalent of the plain text data b. @@ -669,10 +662,6 @@ func JSEscape(w io.Writer, b []byte) { w.Write(jsLt) case '>': w.Write(jsGt) - case '&': - w.Write(jsAmp) - case '=': - w.Write(jsEq) default: w.Write(jsLowUni) t, b := c>>4, c&0x0f @@ -707,7 +696,7 @@ func JSEscapeString(s string) string { func jsIsSpecial(r rune) bool { switch r { - case '\\', '\'', '"', '<', '>', '&', '=': + case '\\', '\'', '"', '<', '>': return true } return r < ' ' || utf8.RuneSelf <= r diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 20c4d1b3a81..0c51d70a66c 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -181,7 +181,7 @@ func (ns *Namespace) ExecuteAsTemplate(args ...interface{}) (resource.Resource, return nil, fmt.Errorf("type %T not supported in Resource transformations", args[2]) } - return ns.templatesClient.ExecuteAsTemplate(r, targetPath, data) + return ns.templatesClient.ExecuteAsTemplate(r, targetPath, data) // TODO1 } // Fingerprint transforms the given Resource with a MD5 hash of the content in diff --git a/tpl/template.go b/tpl/template.go index dba1acd23fe..25e175dd755 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -184,7 +184,10 @@ func (t *TemplateAdapter) extractIdentifiers(line string) []string { return identifiers } +var tplErrCleaner = strings.NewReplacer("invokeDot . ", "", "error calling invokeDot:", "") + func (t *TemplateAdapter) addFileContext(name string, inerr error) error { + inerr = errors.New(tplErrCleaner.Replace(inerr.Error())) if strings.HasPrefix(t.Name(), "_internal") { return inerr } @@ -268,7 +271,7 @@ func (t *TemplateAdapter) ExecuteToString(data interface{}) (string, error) { b := bp.GetBuffer() defer bp.PutBuffer(b) if err := t.Execute(b, data); err != nil { - return "", err + return "", err //TODO1 } return b.String(), nil } diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 2f21c86116f..7b444bb09a1 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -485,7 +485,7 @@ func (t *templateHandler) setFuncs(funcMap map[string]interface{}) { } // SetFuncs replaces the funcs in the func maps with new definitions. -// This is only used in tests. +// This is only used in tests. func (t *templateHandler) SetFuncs(funcMap map[string]interface{}) { t.setFuncs(funcMap) } diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index d4dabcf8268..80aabc24743 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -14,7 +14,6 @@ package tplimpl import ( - "fmt" "html/template" "regexp" "strings" @@ -186,11 +185,13 @@ func applyTemplateTransformers( const ( partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}` dotContextWrapperTempl = `{{ invokeDot . "NAME" "ARGS" }}` + pipeWrapperTempl = `{{ ("ARGS") }}` ) var ( partialReturnWrapper *parse.ListNode dotContextWrapper *parse.CommandNode + pipeWrapper *parse.PipeNode ) func init() { @@ -210,6 +211,12 @@ func init() { } action := templ.Tree.Root.Nodes[0].(*parse.ActionNode) dotContextWrapper = action.Pipe.Cmds[0] + + templ, err = tt.Parse(pipeWrapperTempl) + if err != nil { + panic(err) + } + pipeWrapper = templ.Tree.Root.Nodes[0].(*parse.ActionNode).Pipe } func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.ListNode { @@ -228,58 +235,80 @@ func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.L } var ( - ignoreFuncsRe = regexp.MustCompile("invokeDot|html") - goBuiltInFuncs = regexp.MustCompile("len") + ignoreFuncsRe = regexp.MustCompile("invokeDot|return|html") ) +func (c *templateContext) wrapInPipe(n parse.Node) *parse.PipeNode { + pipe := pipeWrapper.Copy().(*parse.PipeNode) + pipe.Cmds[0].Args = []parse.Node{n} + return pipe +} + +func (c *templateContext) wrapDotIfNeeded(n parse.Node) parse.Node { + switch node := n.(type) { + case *parse.ChainNode: + if pipe, ok := node.Node.(*parse.PipeNode); ok { + for _, cmd := range pipe.Cmds { + c.wrapDot(cmd) + } + } + case *parse.IdentifierNode: + // A function. + if ignoreFuncsRe.MatchString(node.Ident) { + return n + } + + case *parse.PipeNode: + for _, cmd := range node.Cmds { + c.wrapDot(cmd) + } + return n + case *parse.VariableNode: + if len(node.Ident) < 2 { + return n + } + return c.wrapInPipe(n) + default: + } + + return n +} + // TODO1 somehow inject (wrap dot?) a receiver object that can be // set in .Execute. To avoid global funcs. -func (c *templateContext) wrapDot(d bool, cmd *parse.CommandNode) { +func (c *templateContext) wrapDot(cmd *parse.CommandNode) { var dotNode parse.Node - doDebug := d || strings.Contains(cmd.String(), "blue") var fields string firstWord := cmd.Args[0] + c.wrapDotIfNeeded(firstWord) + switch a := firstWord.(type) { case *parse.FieldNode: fields = a.String() - //return s.evalFieldNode(dot, n, cmd.Args, final) case *parse.ChainNode: - if pipe, ok := a.Node.(*parse.PipeNode); ok { - for _, cmd := range pipe.Cmds { - c.wrapDot(doDebug, cmd) - } - - } return // TODO1 // fields = a.String() - //return s.evalChainNode(dot, n, cmd.Args, final) case *parse.IdentifierNode: // Must be a function. if ignoreFuncsRe.MatchString(a.Ident) { return } - if goBuiltInFuncs.MatchString(a.Ident) { - fmt.Println(a.Ident, "==>", cmd.Args[1:]) - return - } fields = a.Ident case *parse.PipeNode: - for _, cmd := range a.Cmds { - c.wrapDot(doDebug, cmd) - } - //s.notAFunction(cmd.Args, final) - //return s.evalPipeline(dot, n) return case *parse.VariableNode: + if len(a.Ident) < 2 { + return + } + // $x.Field has $x as the first ident, Field as the second. fields = "." + strings.Join(a.Ident[1:], ".") a.Ident = a.Ident[:1] dotNode = a default: - //fmt.Printf("UNKNOWN: %T\n", firstWord) return } @@ -295,6 +324,9 @@ func (c *templateContext) wrapDot(d bool, cmd *parse.CommandNode) { args := wrapper.Args[:3] if len(cmd.Args) > 1 { + for i, n := range cmd.Args[1:] { + cmd.Args[i+1] = c.wrapDotIfNeeded(n) + } args = append(args, cmd.Args[1:]...) } @@ -372,7 +404,8 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.CommandNode: - c.wrapDot(false, x) + c.collectInner(x) + c.wrapDot(x) if false || len(x.Args) > 1 { first := x.Args[0] var id string @@ -399,7 +432,6 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } } - c.collectInner(x) keep := c.collectReturnNode(x) for _, elem := range x.Args { diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index a9e9be9f3b8..8e0c45e7004 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -69,7 +69,13 @@ func (*TemplateProvider) CreateFuncMap(in interface{}) map[string]interface{} { } } + } + // Add the Go internal funcs that's not overloaded above. + for k, v := range internal.Builtins { + if _, exists := funcMap[k]; !exists { + funcMap[k] = v + } } return funcMap From 5604f5de3746aa2a6de9caca39ac85dcef48ebd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 19:31:15 +0100 Subject: [PATCH 15/19] Speed --- common/hreflect/invoke.go | 8 ++++---- deps/deps.go | 3 ++- tpl/compare/compare.go | 5 ++--- tpl/tplimpl/templateProvider.go | 17 +++++++++++++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go index ea2c0418692..34adab6461b 100644 --- a/common/hreflect/invoke.go +++ b/common/hreflect/invoke.go @@ -31,20 +31,20 @@ var ( ) type Invoker struct { - funcs func(name string) interface{} + funcs func(name string) reflect.Value } -func NewInvoker(funcs func(name string) interface{}) *Invoker { +func NewInvoker(funcs func(name string) reflect.Value) *Invoker { return &Invoker{funcs: funcs} } func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{}, error) { name := path[0] f := i.funcs(name) // TODO1 store them as reflect.Value - if f == nil { + if f == zero { return err("function with name %s not found", name) } - result, err := i.invoke(reflect.ValueOf(f), path, args) + result, err := i.invoke(f, path, args) if err != nil { return nil, err } diff --git a/deps/deps.go b/deps/deps.go index 4cc131d2931..08b6082a0c0 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -1,6 +1,7 @@ package deps import ( + "reflect" "sync" "time" @@ -40,7 +41,7 @@ type Deps struct { // The templates to use. This will usually implement the full tpl.TemplateHandler. Tmpl tpl.TemplateFinder `json:"-"` - TemplateFuncs map[string]interface{} `json:"-"` + TemplateFuncs map[string]reflect.Value `json:"-"` // We use this to parse and execute ad-hoc text templates. TextTmpl tpl.TemplateParseFinder `json:"-"` diff --git a/tpl/compare/compare.go b/tpl/compare/compare.go index 8daf88fecf2..d82992c24d2 100644 --- a/tpl/compare/compare.go +++ b/tpl/compare/compare.go @@ -30,15 +30,14 @@ import ( // New returns a new instance of the compare-namespaced template functions. func New(deps *deps.Deps, caseInsensitive bool) *Namespace { - var funcs func(name string) interface{} var invoker *hreflect.Invoker if deps != nil { - funcs = func(name string) interface{} { + funcs := func(name string) reflect.Value { funcm := deps.TemplateFuncs if fn, ok := funcm[name]; ok { return fn } - return nil + return reflect.Value{} } invoker = hreflect.NewInvoker(funcs) diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go index 75abe901abe..2be41cba081 100644 --- a/tpl/tplimpl/templateProvider.go +++ b/tpl/tplimpl/templateProvider.go @@ -14,6 +14,8 @@ package tplimpl import ( + "reflect" + "github.com/gohugoio/hugo/deps" ) @@ -29,7 +31,12 @@ func (p *TemplateProvider) Update(deps *deps.Deps) error { newTmpl := newTemplateAdapter(deps) deps.Tmpl = newTmpl deps.TextTmpl = newTmpl.wrapTextTemplate(newTmpl.text.standalone) - deps.TemplateFuncs = p.CreateFuncMap(deps) + funcs := p.CreateFuncMap(deps) + funcsv := make(map[string]reflect.Value) + for k, v := range funcs { + funcsv[k] = reflect.ValueOf(v) + } + deps.TemplateFuncs = funcsv newTmpl.initFuncs() @@ -55,7 +62,13 @@ func (p *TemplateProvider) Clone(d *deps.Deps) error { t := d.Tmpl.(*templateHandler) clone := t.clone(d) - d.TemplateFuncs = p.CreateFuncMap(d) + funcs := p.CreateFuncMap(d) + funcsv := make(map[string]reflect.Value) + for k, v := range funcs { + funcsv[k] = reflect.ValueOf(v) + } + + d.TemplateFuncs = funcsv return clone.MarkReady() From 9867287bf4b994586b113e61e03388e77f3c7c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 21:15:53 +0100 Subject: [PATCH 16/19] Work --- common/hreflect/invoke.go | 63 ++++++++++++++++++++++++++++++++++----- tpl/compare/compare.go | 4 +-- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/common/hreflect/invoke.go b/common/hreflect/invoke.go index 34adab6461b..ddff002ed9b 100644 --- a/common/hreflect/invoke.go +++ b/common/hreflect/invoke.go @@ -17,6 +17,7 @@ package hreflect import ( "fmt" + "html/template" "reflect" "github.com/gohugoio/hugo/common/maps" @@ -30,15 +31,20 @@ var ( reflectValueType = reflect.TypeOf((*reflect.Value)(nil)).Elem() ) -type Invoker struct { - funcs func(name string) reflect.Value +type Invoker interface { + Invoke(name string, args ...interface{}) (interface{}, error, bool) } -func NewInvoker(funcs func(name string) reflect.Value) *Invoker { - return &Invoker{funcs: funcs} +type InvokeManager struct { + invoker Invoker + funcs func(name string) reflect.Value } -func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{}, error) { +func NewInvoker(funcs func(name string) reflect.Value) *InvokeManager { + return &InvokeManager{funcs: funcs} +} + +func (i *InvokeManager) InvokeFunction(path []string, args ...interface{}) (interface{}, error) { name := path[0] f := i.funcs(name) // TODO1 store them as reflect.Value if f == zero { @@ -64,9 +70,50 @@ func (i *Invoker) InvokeFunction(path []string, args ...interface{}) (interface{ return result.Interface(), nil } -func (i *Invoker) InvokeMethod(receiver interface{}, path []string, args ...interface{}) (interface{}, error) { +type titler interface { + Title() string + IsHome() bool + IsTranslated() bool + Permalink() string + Kind() string + Summary() template.HTML + + // RelPermalink represents the host relative link to this resource. + RelPermalink() string + Content() (interface{}, error) +} + +func (i *InvokeManager) InvokeMethod(receiver interface{}, path []string, args ...interface{}) (interface{}, error) { var v reflect.Value + if len(path) == 1 { + if p, ok := receiver.(titler); ok { + name := path[0] + + switch name { + case "Title": + return p.Title(), nil + case "Permalink": + return p.Permalink(), nil + case "Kind": + return p.Kind(), nil + case "RelPermalink": + return p.RelPermalink(), nil + case "IsTranslated": + return p.IsTranslated(), nil + case "IsHome": + return p.IsHome(), nil + case "Content": + return p.Content() + case "Summary": + return p.Summary(), nil + default: + //fmt.Println("::", name) + } + + } + } + if rv, ok := receiver.(reflect.Value); ok { v = rv } else { @@ -125,7 +172,7 @@ func argsToValues(args []interface{}, typ reflect.Type) []reflect.Value { return argsv } -func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface{}) (reflect.Value, error) { +func (i *InvokeManager) invoke(receiver reflect.Value, path []string, args []interface{}) (reflect.Value, error) { if len(path) == 0 { return receiver, nil } @@ -201,7 +248,7 @@ func (i *Invoker) invoke(receiver reflect.Value, path []string, args []interface return receiver, nil } -func (i *Invoker) call(fun reflect.Value, args []reflect.Value) (val reflect.Value, err error) { +func (i *InvokeManager) call(fun reflect.Value, args []reflect.Value) (val reflect.Value, err error) { defer func() { if r := recover(); r != nil { if e, ok := r.(error); ok { diff --git a/tpl/compare/compare.go b/tpl/compare/compare.go index d82992c24d2..abd7f63b1fe 100644 --- a/tpl/compare/compare.go +++ b/tpl/compare/compare.go @@ -30,7 +30,7 @@ import ( // New returns a new instance of the compare-namespaced template functions. func New(deps *deps.Deps, caseInsensitive bool) *Namespace { - var invoker *hreflect.Invoker + var invoker *hreflect.InvokeManager if deps != nil { funcs := func(name string) reflect.Value { funcm := deps.TemplateFuncs @@ -48,7 +48,7 @@ func New(deps *deps.Deps, caseInsensitive bool) *Namespace { // Namespace provides template functions for the "compare" namespace. type Namespace struct { - invoker *hreflect.Invoker + invoker *hreflect.InvokeManager // Enable to do case insensitive string compares. caseInsensitive bool From b70b43654a2e798dca7c5399cec82cd5ee8b9dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 21:57:54 +0100 Subject: [PATCH 17/19] Bench --- tpl/tplimpl/template_ast_transformers_test.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go index 76f6f54977a..2e7c108eae7 100644 --- a/tpl/tplimpl/template_ast_transformers_test.go +++ b/tpl/tplimpl/template_ast_transformers_test.go @@ -604,3 +604,48 @@ func TestPartialReturn(t *testing.T) { } } + +const transformBemchMarkTemplate = ` + +{{ Echo "foo" }} + +` + +func TestBench(t *testing.T) { + c := qt.New(t) + + templ, err := template.New("foo").Funcs(testFuncs).Parse(transformBemchMarkTemplate) + c.Assert(err, qt.IsNil) + + ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templ)) + ctx.applyTransformations(templ.Tree.Root) + + s := templ.Tree.Root.String() + + fmt.Println(s) + +} +func BenchmarkTemplateTransform(b *testing.B) { + templ, err := template.New("foo").Funcs(testFuncs).Parse(transformBemchMarkTemplate) + + if err != nil { + b.Fatal(err) + } + + templates := make([]*template.Template, b.N) + + for i := 0; i < b.N; i++ { + templates[i], err = templ.Clone() + if err != nil { + b.Fatal(err) + } + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + c := newTemplateContext( + newTemplateInfo("test"), createParseTreeLookup(templates[i])) + c.applyTransformations(templ.Tree.Root) + } +} From a094f5253d94333e5b801de8f834abf9215d3b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 9 Dec 2019 22:03:11 +0100 Subject: [PATCH 18/19] Remove unused --- tpl/tplimpl/template_ast_transformers.go | 206 ------------------ tpl/tplimpl/template_ast_transformers_test.go | 200 ----------------- 2 files changed, 406 deletions(-) diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index 80aabc24743..e4922e69fe8 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -26,20 +26,6 @@ import ( "github.com/pkg/errors" ) -// decl keeps track of the variable mappings, i.e. $mysite => .Site etc. -type decl map[string]string -type decln map[string]*parse.CommandNode - -const ( - paramsIdentifier = "Params" -) - -// Containers that may contain Params that we will not touch. -var reservedContainers = map[string]bool{ - // Aka .Site.Data.Params which must stay case sensitive. - "Data": true, -} - type templateType int const ( @@ -49,8 +35,6 @@ const ( ) type templateContext struct { - decl decl - decln decln // TODO1 remove visited map[string]bool templateNotFound map[string]bool identityNotFound map[string]bool @@ -97,8 +81,6 @@ func newTemplateContext(info tpl.Info, lookupFn func(name string) *templateInfoT return &templateContext{ Info: info, lookupFn: lookupFn, - decl: make(map[string]string), - decln: make(map[string]*parse.CommandNode), visited: make(map[string]bool), templateNotFound: make(map[string]bool), identityNotFound: make(map[string]bool), @@ -390,12 +372,6 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { } case *parse.PipeNode: c.collectConfig(x) - if len(x.Decl) == 1 && len(x.Cmds) == 1 { - // maps $site => .Site etc. - c.decl[x.Decl[0].Ident[0]] = x.Cmds[0].String() - c.decln[x.Decl[0].Ident[0]] = x.Cmds[0] - } - for i, cmd := range x.Cmds { keep, _ := c.applyTransformations(cmd) if !keep { @@ -436,17 +412,8 @@ func (c *templateContext) applyTransformations(n parse.Node) (bool, error) { for _, elem := range x.Args { switch an := elem.(type) { - case *parse.FieldNode: - c.updateIdentsIfNeeded(an.Ident) - case *parse.VariableNode: - c.updateIdentsIfNeeded(an.Ident) case *parse.PipeNode: c.applyTransformations(an) - case *parse.ChainNode: - // site.Params... - if len(an.Field) > 1 && an.Field[0] == paramsIdentifier { - c.updateIdentsIfNeeded(an.Field) - } } } return keep, c.err @@ -461,22 +428,6 @@ func (c *templateContext) applyTransformationsToNodes(nodes ...parse.Node) { } } -func (c *templateContext) updateIdentsIfNeeded(idents []string) { - if true { - return // TODO1 remove all this .Params stuff. - } - index := c.decl.indexOfReplacementStart(idents) - - if index == -1 { - return - } - - for i := index; i < len(idents); i++ { - idents[i] = strings.ToLower(idents[i]) - } - -} - func (c *templateContext) hasIdent(idents []string, ident string) bool { for _, id := range idents { if id == ident { @@ -578,160 +529,3 @@ func (c *templateContext) collectReturnNode(n *parse.CommandNode) bool { return false } - -// indexOfReplacementStart will return the index of where to start doing replacement, -// -1 if none needed. -func (d decl) indexOfReplacementStart(idents []string) int { - - l := len(idents) - - if l == 0 { - return -1 - } - - if l == 1 { - first := idents[0] - if first == "" || first == paramsIdentifier || first[0] == '$' { - // This can not be a Params.x - return -1 - } - } - - var lookFurther bool - var needsVarExpansion bool - for _, ident := range idents { - if ident[0] == '$' { - lookFurther = true - needsVarExpansion = true - break - } else if ident == paramsIdentifier { - lookFurther = true - break - } - } - - if !lookFurther { - return -1 - } - - var resolvedIdents []string - - if !needsVarExpansion { - resolvedIdents = idents - } else { - var ok bool - resolvedIdents, ok = d.resolveVariables(idents) - if !ok { - return -1 - } - } - - var paramFound bool - for i, ident := range resolvedIdents { - if ident == paramsIdentifier { - if i > 0 { - container := resolvedIdents[i-1] - if reservedContainers[container] { - // .Data.Params.someKey - return -1 - } - } - - paramFound = true - break - } - } - - if !paramFound { - return -1 - } - - var paramSeen bool - idx := -1 - for i, ident := range idents { - if ident == "" || ident[0] == '$' { - continue - } - - if ident == paramsIdentifier { - paramSeen = true - idx = -1 - - } else { - if paramSeen { - return i - } - if idx == -1 { - idx = i - } - } - } - return idx - -} - -func (d decl) resolveVariables(idents []string) ([]string, bool) { - var ( - replacements []string - replaced []string - ) - - // An Ident can start out as one of - // [Params] [$blue] [$colors.Blue] - // We need to resolve the variables, so - // $blue => [Params Colors Blue] - // etc. - replacements = []string{idents[0]} - - // Loop until there are no more $vars to resolve. - for i := 0; i < len(replacements); i++ { - - if i > 20 { - // bail out - return nil, false - } - - potentialVar := replacements[i] - - if potentialVar == "$" { - continue - } - - if potentialVar == "" || potentialVar[0] != '$' { - // leave it as is - replaced = append(replaced, strings.Split(potentialVar, ".")...) - continue - } - - replacement, ok := d[potentialVar] - - if !ok { - // Temporary range vars. We do not care about those. - return nil, false - } - - if !d.isKeyword(replacement) { - continue - } - - replacement = strings.TrimPrefix(replacement, ".") - - if replacement == "" { - continue - } - - if replacement[0] == '$' { - // Needs further expansion - replacements = append(replacements, strings.Split(replacement, ".")...) - } else { - replaced = append(replaced, strings.Split(replacement, ".")...) - } - } - - return append(replaced, idents[1:]...), true - -} - -func (d decl) isKeyword(s string) bool { - return !strings.ContainsAny(s, " -\"") -} diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go index 2e7c108eae7..e82d06c7b62 100644 --- a/tpl/tplimpl/template_ast_transformers_test.go +++ b/tpl/tplimpl/template_ast_transformers_test.go @@ -13,7 +13,6 @@ package tplimpl import ( - "bytes" "fmt" "html/template" "testing" @@ -215,205 +214,6 @@ PARAMS GETPAGE2: {{ $p.Params.LOWER }} ` ) -func TestParamsKeysToLower(t *testing.T) { - t.Parallel() - c := qt.New(t) - - _, err := applyTemplateTransformers(templateUndefined, nil, nil) - c.Assert(err, qt.Not(qt.IsNil)) - - templ, err := template.New("foo").Funcs(testFuncs).Parse(paramsTempl) - - c.Assert(err, qt.IsNil) - - ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templ)) - - c.Assert(ctx.decl.indexOfReplacementStart([]string{}), qt.Equals, -1) - - ctx.applyTransformations(templ.Tree.Root) - - var b bytes.Buffer - - c.Assert(templ.Execute(&b, paramsData), qt.IsNil) - - result := b.String() - - c.Assert(result, qt.Contains, "P1: P1L") - c.Assert(result, qt.Contains, "P1_2: P1L") - c.Assert(result, qt.Contains, "P1_3: P1L") - c.Assert(result, qt.Contains, "P1_4: P1L") - c.Assert(result, qt.Contains, "P2: P2L") - c.Assert(result, qt.Contains, "P2_2: P2L") - c.Assert(result, qt.Contains, "P2_3: P2L") - c.Assert(result, qt.Contains, "P2_4: P2L") - c.Assert(result, qt.Contains, "P22: P22L") - c.Assert(result, qt.Contains, "P22_nested: P22L_nested") - c.Assert(result, qt.Contains, "P3: P3H") - c.Assert(result, qt.Contains, "P3_2: P3H") - c.Assert(result, qt.Contains, "P3_3: P3H") - c.Assert(result, qt.Contains, "P3_4: P3H") - c.Assert(result, qt.Contains, "P4: 13") - c.Assert(result, qt.Contains, "P5: P1L") - c.Assert(result, qt.Contains, "P5_2: P2L") - - c.Assert(result, qt.Contains, "IF: P1L") - c.Assert(result, qt.Contains, "ELSE: P1L") - - c.Assert(result, qt.Contains, "WITH: P1L") - - c.Assert(result, qt.Contains, "RANGE: 3: P1L") - - c.Assert(result, qt.Contains, "Hi There") - - // Issue #2740 - c.Assert(result, qt.Contains, "F1: themes/P2L-theme") - c.Assert(result, qt.Contains, "F2: themes/P2L-theme") - c.Assert(result, qt.Contains, "F3: themes/P2L-theme") - - c.Assert(result, qt.Contains, "PSLICE: PSLICE1|PSLICE3|") - c.Assert(result, qt.Contains, "PARAMS STRING: foo:.Params.toc_hide:[!= true]") - c.Assert(result, qt.Contains, "PARAMS STRING2: foo:.Params.toc_hide:[!= true]") - c.Assert(result, qt.Contains, "PARAMS STRING3: .Params.TOC_HIDE:!=:[P1L]") - - // Issue #5094 - c.Assert(result, qt.Contains, "PARAMS COMPOSITE: [1 3]") - - // Issue #5068 - c.Assert(result, qt.Contains, "PCurrentSection: pcurrentsection") - - // Issue #5541 - c.Assert(result, qt.Contains, "PARAMS TIME: 1972-02-28") - c.Assert(result, qt.Contains, "PARAMS TIME2: 1972-02-28") - - // Issue ##5615 - c.Assert(result, qt.Contains, "PARAMS SITE GLOBAL1: global-site") - c.Assert(result, qt.Contains, "PARAMS SITE GLOBAL2: global-site") - c.Assert(result, qt.Contains, "PARAMS SITE GLOBAL3: global-site") - - // - c.Assert(result, qt.Contains, "PARAMS GETPAGE: page") - c.Assert(result, qt.Contains, "PARAMS GETPAGE2: page") - -} - -func BenchmarkTemplateParamsKeysToLower(b *testing.B) { - templ, err := template.New("foo").Funcs(testFuncs).Parse(paramsTempl) - - if err != nil { - b.Fatal(err) - } - - templates := make([]*template.Template, b.N) - - for i := 0; i < b.N; i++ { - templates[i], err = templ.Clone() - if err != nil { - b.Fatal(err) - } - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - c := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templates[i])) - c.applyTransformations(templ.Tree.Root) - } -} - -func TestParamsKeysToLowerVars(t *testing.T) { - t.Parallel() - c := qt.New(t) - - var ( - data = map[string]interface{}{ - "Params": map[string]interface{}{ - "colors": map[string]interface{}{ - "blue": "Amber", - "pretty": map[string]interface{}{ - "first": "Indigo", - }, - }, - }, - } - - // This is how Amber behaves: - paramsTempl = ` -{{$__amber_1 := .Params.Colors}} -{{$__amber_2 := $__amber_1.Blue}} -{{$__amber_3 := $__amber_1.Pretty}} -{{$__amber_4 := .Params}} - -Color: {{$__amber_2}} -Blue: {{ $__amber_1.Blue}} -Pretty First1: {{ $__amber_3.First}} -Pretty First2: {{ $__amber_1.Pretty.First}} -Pretty First3: {{ $__amber_4.COLORS.PRETTY.FIRST}} -` - ) - - templ, err := template.New("foo").Parse(paramsTempl) - - c.Assert(err, qt.IsNil) - - ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(templ)) - - ctx.applyTransformations(templ.Tree.Root) - - var b bytes.Buffer - - c.Assert(templ.Execute(&b, data), qt.IsNil) - - result := b.String() - - c.Assert(result, qt.Contains, "Color: Amber") - c.Assert(result, qt.Contains, "Blue: Amber") - c.Assert(result, qt.Contains, "Pretty First1: Indigo") - c.Assert(result, qt.Contains, "Pretty First2: Indigo") - c.Assert(result, qt.Contains, "Pretty First3: Indigo") - -} - -func TestParamsKeysToLowerInBlockTemplate(t *testing.T) { - t.Parallel() - c := qt.New(t) - - var ( - data = map[string]interface{}{ - "Params": map[string]interface{}{ - "lower": "P1L", - }, - } - - master = ` -P1: {{ .Params.LOWER }} -{{ block "main" . }}DEFAULT{{ end }}` - overlay = ` -{{ define "main" }} -P2: {{ .Params.LOWER }} -{{ end }}` - ) - - masterTpl, err := template.New("foo").Parse(master) - c.Assert(err, qt.IsNil) - - overlayTpl, err := template.Must(masterTpl.Clone()).Parse(overlay) - c.Assert(err, qt.IsNil) - overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) - - ctx := newTemplateContext(newTemplateInfo("test"), createParseTreeLookup(overlayTpl)) - - ctx.applyTransformations(overlayTpl.Tree.Root) - - var b bytes.Buffer - - c.Assert(overlayTpl.Execute(&b, data), qt.IsNil) - - result := b.String() - - c.Assert(result, qt.Contains, "P1: P1L") - c.Assert(result, qt.Contains, "P2: P1L") -} - // Issue #2927 func TestTransformRecursiveTemplate(t *testing.T) { c := qt.New(t) From f28cbb59a820d341a1f733b7dcbef680b1a0b5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 10 Dec 2019 07:44:03 +0100 Subject: [PATCH 19/19] Work --- tpl/tplimpl/template_ast_transformers.go | 29 +++++++++++++------ tpl/tplimpl/template_ast_transformers_test.go | 2 ++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tpl/tplimpl/template_ast_transformers.go b/tpl/tplimpl/template_ast_transformers.go index e4922e69fe8..8f1383f8ee3 100644 --- a/tpl/tplimpl/template_ast_transformers.go +++ b/tpl/tplimpl/template_ast_transformers.go @@ -14,6 +14,7 @@ package tplimpl import ( + "fmt" "html/template" "regexp" "strings" @@ -168,11 +169,13 @@ const ( partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}` dotContextWrapperTempl = `{{ invokeDot . "NAME" "ARGS" }}` pipeWrapperTempl = `{{ ("ARGS") }}` + // Consder creating these with the current template ) var ( partialReturnWrapper *parse.ListNode dotContextWrapper *parse.CommandNode + dotContextFunc *parse.IdentifierNode pipeWrapper *parse.PipeNode ) @@ -193,6 +196,7 @@ func init() { } action := templ.Tree.Root.Nodes[0].(*parse.ActionNode) dotContextWrapper = action.Pipe.Cmds[0] + dotContextFunc = dotContextWrapper.Args[0].(*parse.IdentifierNode) templ, err = tt.Parse(pipeWrapperTempl) if err != nil { @@ -223,6 +227,7 @@ var ( func (c *templateContext) wrapInPipe(n parse.Node) *parse.PipeNode { pipe := pipeWrapper.Copy().(*parse.PipeNode) pipe.Cmds[0].Args = []parse.Node{n} + return pipe } @@ -236,10 +241,6 @@ func (c *templateContext) wrapDotIfNeeded(n parse.Node) parse.Node { } case *parse.IdentifierNode: // A function. - if ignoreFuncsRe.MatchString(node.Ident) { - return n - } - case *parse.PipeNode: for _, cmd := range node.Cmds { c.wrapDot(cmd) @@ -249,7 +250,11 @@ func (c *templateContext) wrapDotIfNeeded(n parse.Node) parse.Node { if len(node.Ident) < 2 { return n } - return c.wrapInPipe(n) + fmt.Println(">>>N", node) + n = c.wrapInPipe(n) + fmt.Println(">>>", n) + n = c.wrapDotIfNeeded(n) + fmt.Println(">>>2", n) default: } @@ -264,8 +269,6 @@ func (c *templateContext) wrapDot(cmd *parse.CommandNode) { firstWord := cmd.Args[0] - c.wrapDotIfNeeded(firstWord) - switch a := firstWord.(type) { case *parse.FieldNode: fields = a.String() @@ -273,11 +276,19 @@ func (c *templateContext) wrapDot(cmd *parse.CommandNode) { return // TODO1 // fields = a.String() case *parse.IdentifierNode: - // Must be a function. + // A function. if ignoreFuncsRe.MatchString(a.Ident) { return } - fields = a.Ident + args := make([]parse.Node, len(cmd.Args)+1) + args[0] = dotContextFunc + for i := 1; i < len(args); i++ { + args[i] = cmd.Args[i-1] + c.wrapDotIfNeeded(args[i]) + } + cmd.Args = args + //fmt.Println("wrapDot", fields) + return case *parse.PipeNode: return case *parse.VariableNode: diff --git a/tpl/tplimpl/template_ast_transformers_test.go b/tpl/tplimpl/template_ast_transformers_test.go index e82d06c7b62..9ca8a35101e 100644 --- a/tpl/tplimpl/template_ast_transformers_test.go +++ b/tpl/tplimpl/template_ast_transformers_test.go @@ -407,7 +407,9 @@ func TestPartialReturn(t *testing.T) { const transformBemchMarkTemplate = ` +{{ $params := "foo" }} {{ Echo "foo" }} +{{ Echo $params.Foo }} `