From 58f833f580a6a78f4cbad0b3c159322add08ba9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 27 Jun 2024 16:22:35 +0200 Subject: [PATCH] Add js.Batch Fixes #12626 Closes #7499 Closes #12874 --- commands/commandeer.go | 1 + common/herrors/errors.go | 15 + common/maps/scratch.go | 17 + common/types/closer.go | 7 + debug.log | 0 deps/deps.go | 15 + hugolib/hugo_sites.go | 4 + hugolib/integrationtest_builder.go | 21 +- hugolib/pages_capture.go | 10 +- hugolib/rebuild_test.go | 12 + hugolib/site.go | 6 +- internal/js/esbuild/batch-esm-runner.gotmpl | 22 + internal/js/esbuild/batch.go | 1137 +++++++++++++++++ internal/js/esbuild/batch_integration_test.go | 428 +++++++ internal/js/esbuild/build.go | 181 +++ internal/js/esbuild/options.go | 301 +++++ internal/js/esbuild/options_test.go | 220 ++++ internal/js/esbuild/resolve.go | 275 ++++ internal/js/esbuild/resolve_test.go | 85 ++ lazy/init.go | 2 +- lazy/once.go | 10 +- media/mediaType.go | 12 +- output/outputFormat.go | 9 + resources/page/page.go | 3 +- resources/resource.go | 6 + resources/resource/resources.go | 157 ++- resources/resource/resources_test.go | 125 ++ resources/resource/resourcetypes.go | 15 + resources/resource_transformers/js/build.go | 195 +-- .../resource_transformers/js/build_test.go | 14 - .../js/js_integration_test.go | 2 +- resources/resource_transformers/js/options.go | 461 ------- .../resource_transformers/js/options_test.go | 241 ---- .../resource_transformers/js/transform.go | 68 + resources/transform.go | 6 + tpl/js/init.go | 5 +- tpl/js/js.go | 47 +- .../_hugo/build/js/batch-esm-runner.gotmpl | 16 + 38 files changed, 3230 insertions(+), 921 deletions(-) create mode 100644 debug.log create mode 100644 internal/js/esbuild/batch-esm-runner.gotmpl create mode 100644 internal/js/esbuild/batch.go create mode 100644 internal/js/esbuild/batch_integration_test.go create mode 100644 internal/js/esbuild/build.go create mode 100644 internal/js/esbuild/options.go create mode 100644 internal/js/esbuild/options_test.go create mode 100644 internal/js/esbuild/resolve.go create mode 100644 internal/js/esbuild/resolve_test.go create mode 100644 resources/resource/resources_test.go delete mode 100644 resources/resource_transformers/js/build_test.go delete mode 100644 resources/resource_transformers/js/options.go delete mode 100644 resources/resource_transformers/js/options_test.go create mode 100644 resources/resource_transformers/js/transform.go create mode 100644 tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl diff --git a/commands/commandeer.go b/commands/commandeer.go index ad2adf3a289..96ffce7d6c3 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -455,6 +455,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error { r.hugoSites = lazycache.New(lazycache.Options[configKey, *hugolib.HugoSites]{ MaxEntries: 1, OnEvict: func(key configKey, value *hugolib.HugoSites) { + fmt.Println("Evicting HugoSites", key) // TODO1 remove me. value.Close() runtime.GC() }, diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 40833e55c04..c7ee90dd0ec 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -133,6 +133,21 @@ func IsNotExist(err error) bool { return false } +// IsExist returns true if the error is a file exists error. +// Unlike os.IsExist, this also considers wrapped errors. +func IsExist(err error) bool { + if os.IsExist(err) { + return true + } + + // os.IsExist does not consider wrapped errors. + if os.IsExist(errors.Unwrap(err)) { + return true + } + + return false +} + var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`) const deferredPrefix = "__hdeferred/" diff --git a/common/maps/scratch.go b/common/maps/scratch.go index e9f412540b2..0d47f452c60 100644 --- a/common/maps/scratch.go +++ b/common/maps/scratch.go @@ -107,6 +107,23 @@ func (c *Scratch) Get(key string) any { return val } +// GetOrCreate returns the value for the given key if it exists, or creates it +// using the given func and stores that value in the map. +// For internal use. +func (c *Scratch) GetOrCreate(key string, create func() (any, error)) (any, error) { + c.mu.Lock() + defer c.mu.Unlock() + if val, found := c.values[key]; found { + return val, nil + } + val, err := create() + if err != nil { + return nil, err + } + c.values[key] = val + return val, nil +} + // Values returns the raw backing map. Note that you should just use // this method on the locally scoped Scratch instances you obtain via newScratch, not // .Page.Scratch etc., as that will lead to concurrency issues. diff --git a/common/types/closer.go b/common/types/closer.go index 2844b1986ef..9f8875a8a2f 100644 --- a/common/types/closer.go +++ b/common/types/closer.go @@ -19,6 +19,13 @@ type Closer interface { Close() error } +// CloserFunc is a convenience type to create a Closer from a function. +type CloserFunc func() error + +func (f CloserFunc) Close() error { + return f() +} + type CloseAdder interface { Add(Closer) } diff --git a/debug.log b/debug.log new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deps/deps.go b/deps/deps.go index 4389036cbdf..aa24ce4f216 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "os" "path/filepath" "sort" "strings" @@ -268,6 +269,20 @@ func (d *Deps) Compile(prototype *Deps) error { return nil } +// MkdirTemp returns a temporary directory path that will be cleaned up on exit. +func (d Deps) MkdirTemp(pattern string) (string, error) { + filename, err := os.MkdirTemp("", pattern) + if err != nil { + return "", err + } + d.BuildClosers.Add(types.CloserFunc( + func() error { + return os.RemoveAll(filename) + })) + + return filename, nil +} + type globalErrHandler struct { logger loggers.Logger diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 659a772f222..1cab8fead5e 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -111,6 +111,10 @@ func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { return h.skipRebuildForFilenames[ev.Name] } +func (h *HugoSites) Close() error { + return h.Deps.Close() +} + func (h *HugoSites) isRebuild() bool { return h.buildCounter.Load() > 0 } diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index b45defb42d2..1537f27ec93 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -81,6 +81,13 @@ func TestOptWithNFDOnDarwin() TestOpt { } } +// TestOptWithOSFs enables the real file system. +func TestOptWithOSFs() TestOpt { + return func(c *IntegrationTestConfig) { + c.NeedsOsFS = true + } +} + // TestOptWithWorkingDir allows setting any config optiona as a function al option. func TestOptWithConfig(fn func(c *IntegrationTestConfig)) TestOpt { return func(c *IntegrationTestConfig) { @@ -280,8 +287,9 @@ func (s *IntegrationTestBuilder) negate(match string) (string, bool) { func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) { s.Helper() content := strings.TrimSpace(s.FileContent(filename)) + for _, m := range matches { - cm := qt.Commentf("File: %s Match %s", filename, m) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) lines := strings.Split(m, "\n") for _, match := range lines { match = strings.TrimSpace(match) @@ -291,7 +299,8 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s var negate bool match, negate = s.negate(match) if negate { - s.Assert(content, qt.Not(qt.Contains), match, cm) + if !s.Assert(content, qt.Not(qt.Contains), match, cm) { + } continue } s.Assert(content, qt.Contains, match, cm) @@ -303,7 +312,8 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches s.Helper() content := s.FileContent(filename) for _, m := range matches { - s.Assert(content, qt.Contains, m, qt.Commentf(m)) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) + s.Assert(content, qt.Contains, m, cm) } } @@ -430,6 +440,11 @@ func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder { return s } +func (s *IntegrationTestBuilder) Close() { + s.Helper() + s.Assert(s.H.Close(), qt.IsNil) +} + func (s *IntegrationTestBuilder) LogString() string { return s.lastBuildLog } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 96c2c0f96e2..3993e7ff33d 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -149,7 +149,15 @@ func (c *pagesCollector) Collect() (collectErr error) { id.p, false, func(fim hugofs.FileMetaInfo) bool { - return true + if id.isStructuralChange() { + return true + } + fimp := fim.Meta().PathInfo + if fimp == nil { + return true + } + + return fimp.Path() == id.p.Path() }, ) } else if id.p.IsBranchBundle() { diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index 2219fe81226..e3d7c79525b 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -71,6 +71,18 @@ Foo. ` +func TestRebuildEditLeafBundleHeaderOnly(t *testing.T) { + b := TestRunning(t, rebuildFilesSimple) + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Content.") + + b.EditFileReplaceAll("content/mysection/mysectionbundle/index.md", "My Section Bundle Content.", "My Section Bundle Content Edited.").Build() + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Edited.") + b.AssertRenderCountPage(1) + b.AssertRenderCountContent(1) +} + func TestRebuildEditTextFileInLeafBundle(t *testing.T) { b := TestRunning(t, rebuildFilesSimple) b.AssertFileContent("public/mysection/mysectionbundle/index.html", diff --git a/hugolib/site.go b/hugolib/site.go index 08031390ba1..242a2b9812c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1522,7 +1522,11 @@ func (s *Site) renderForTemplate(ctx context.Context, name, outputFormat string, } if err = s.Tmpl().ExecuteWithContext(ctx, templ, w, d); err != nil { - return fmt.Errorf("render of %q failed: %w", name, err) + filename := name + if p, ok := d.(*pageState); ok { + filename = p.pathOrTitle() + } + return fmt.Errorf("render of %q failed: %w", filename, err) } return } diff --git a/internal/js/esbuild/batch-esm-runner.gotmpl b/internal/js/esbuild/batch-esm-runner.gotmpl new file mode 100644 index 00000000000..bba13b00c5a --- /dev/null +++ b/internal/js/esbuild/batch-esm-runner.gotmpl @@ -0,0 +1,22 @@ +{{ range $i, $e := .Scripts -}} + {{ if eq .Export "*" }} + {{ printf "import %s as Script%d from %q;" .Export $i .Import }} + {{ else }} + {{ printf "import { %s as Script%d } from %q;" .Export $i .Import }} + {{ end }} +{{ end -}} +{{ range $i, $e := .Runners }} + {{ printf "import { %s as Run%d } from %q;" .Export $i .Import }} +{{ end }} +{{/* */}} +{{ if .Runners }} + let scripts = []; + {{ range $i, $e := .Scripts -}} + scripts.push({{ .RunnerJSON $i }}); + {{ end -}} + {{/* */}} + {{ range $i, $e := .Runners }} + {{ $id := printf "Run%d" $i }} + {{ $id }}(scripts); + {{ end }} +{{ end }} diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go new file mode 100644 index 00000000000..787eb9c4e2f --- /dev/null +++ b/internal/js/esbuild/batch.go @@ -0,0 +1,1137 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package esbuild provides functions for building JavaScript resources. +package esbuild + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/lazy" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" + "github.com/gohugoio/hugo/tpl" + "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" +) + +var _ Batcher = (*batcher)(nil) + +var _ resource.StaleInfo = (*options)(nil) + +var _ identity.Identity = (*Package)(nil) + +const ( + NsBatch = "__hugo-js-batch" +) + +func (g *getOnce[T]) Get() T { + var v T + g.once.Do(func() { + v = g.v + }) + return v +} + +func newOptions() *options { + return &options{getOnce[*optionsSetter]{ + v: &optionsSetter{}, + }} +} + +type Batcher interface { + Build(context.Context) (*Package, error) + Config() OptionsSetter + Group(id string) BatcherGroup +} + +//go:embed batch-esm-runner.gotmpl +var runnerTemplateStr string + +func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) { + return &BatcherClient{ + d: deps, + buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec), + createClient: create.New(deps.ResourceSpec), + bundlesCache: dynacache.GetOrCreatePartition[string, *Package]( + deps.MemCache, + "/jsb1", + // Mark it to clear on rebuild, but each package will evaluate itself for changes. + dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnRebuild, Weight: 10}, + ), + }, nil +} + +type BatcherClient struct { + d *deps.Deps + + once sync.Once + runnerTemplate tpl.Template + + createClient *create.Client + buildClient *BuildClient + + bundlesCache *dynacache.Partition[string, *Package] +} + +// New creates a new Batcher with the given ID. +func (c *BatcherClient) New(id string) (Batcher, error) { + var initErr error + c.once.Do(func() { + // We should fix the initialization order here (or use the Go template package directly), but we need to wait + // for the Hugo templates to be ready. + tmpl, err := c.d.TextTmpl().Parse("batch-esm-runner", runnerTemplateStr) + if err != nil { + initErr = err + return + } + c.runnerTemplate = tmpl + }) + + if initErr != nil { + return nil, initErr + } + + return &batcher{ + id: id, + scriptGroups: make(map[string]*scriptGroup), + dependencyManager: c.d.Conf.NewIdentityManager("jsbatch" + id), + client: c, + configOptions: newOptions(), + }, nil +} + +func (c *BatcherClient) buildBatch(ctx context.Context, t *batchTemplateContext) (resource.Resource, string, error) { + var buf bytes.Buffer + + if err := c.d.Tmpl().ExecuteWithContext(ctx, c.runnerTemplate, &buf, t); err != nil { + return nil, "", err + } + + s := paths.AddLeadingSlash(t.keyPath + ".js") + r, err := c.createClient.FromString(s, buf.String()) + if err != nil { + return nil, "", err + } + + return r, s, nil +} + +type BatcherGroup interface { + Instance(sid, iid string) OptionsSetter + Runner(id string) OptionsSetter + Script(id string) OptionsSetter +} + +type OptionsSetter interface { + SetOptions(map[string]any) string +} + +// TODO1 names. +type Package struct { + origin *batcher + outDir string + id string + staleVersion uint32 + b *batcher + Groups map[string]resource.Resources +} + +func (b *Package) GetDependencyManager() identity.Manager { + return b.origin.dependencyManager +} + +func (p *Package) IdentifierBase() string { + return p.id +} + +func (p *Package) MarkStale() { + p.origin.reset() +} + +func (p *Package) calculateStaleVersion() uint32 { + // Return the first non-zero value found. + var i uint32 + p.forEeachStaleInfo(func(si resource.StaleInfo) bool { + if i = si.StaleVersion(); i > 0 { + return true + } + return false + }) + + return i +} + +// You should not depend on the invocation order when calling this. +// TODO1 check that this does not get called on first build. +func (p *Package) forEeachStaleInfo(f func(si resource.StaleInfo) bool) { + check := func(v any) bool { + if si, ok := v.(resource.StaleInfo); ok { + return f(si) + } + return false + } + for _, v := range p.b.scriptGroups { + if b := func() bool { + v.mu.Lock() + defer v.mu.Unlock() + + for _, vv := range v.instancesOptions { + if check(vv) { + return true + } + } + + for _, vv := range v.scriptsOptions { + if check(vv) { + return true + } + } + + for _, vv := range v.runnersOptions { + if check(vv) { + return true + } + } + + return false + }(); b { + return + } + } +} + +type ParamsOptions struct { + Params json.RawMessage +} + +type ScriptOptions struct { + // The script to build. + // TODO1 handle stale. + Resource resource.Resource + + // The import context to use. + // Note that we will always fall back to the resource's own import context. + ImportContext resource.ResourceGetter + + // The export name to use for this script's group's runners (if any). + // If not set, the default export will be used. + Export string + + // Params marshaled to JSON. + Params json.RawMessage +} + +func (o ScriptOptions) Compile(m map[string]any) (*ScriptOptions, error) { + var s optionsGetSet // TODO1 type. + if err := mapstructure.WeakDecode(m, &s); err != nil { + return nil, err + } + + paramsJSON, err := json.Marshal(s.Params) + if err != nil { + panic(err) + } + + return &ScriptOptions{ + Resource: s.Resource, + ImportContext: resource.NewResourceGetter(s.ImportContext), + Params: paramsJSON, + }, nil +} + +func (o *ScriptOptions) Dir() string { + return path.Dir(o.Resource.(resource.PathProvider).Path()) +} + +type ScriptOptionsGetSetter interface { + GetOptions() *ScriptOptions + SetOptions(map[string]any) string +} + +type batchTemplateContext struct { + keyPath string + ID string + Runners []scriptRunnerTemplateContext + Scripts []scriptBatchTemplateContext +} + +type batcher struct { + mu sync.Mutex + id string + scriptGroups scriptGroups + + client *BatcherClient + dependencyManager identity.Manager + + configOptions *options + + // The last successfully built package. + // If this is non-nil and not stale, we can reuse it (e.g. on server rebuilds) + prevBuild *Package + + // Compiled. + config ExternalOptions +} + +func (b *batcher) Build(ctx context.Context) (*Package, error) { + key := dynacache.CleanKey(b.id + ".js") + p, err := b.client.bundlesCache.GetOrCreateWitTimeout(key, b.client.d.Conf.Timeout(), func(string) (*Package, error) { + return b.build(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to build Batch %q: %w", b.id, err) + } + + if p.b != b { + panic("bundler mismatch") + } + + return p, nil +} + +func (b *batcher) Config() OptionsSetter { + return b.configOptions.Get() +} + +func (b *batcher) Group(id string) BatcherGroup { + b.mu.Lock() + defer b.mu.Unlock() + + group, found := b.scriptGroups[id] + if !found { + group = &scriptGroup{ + id: id, client: b.client, + scriptsOptions: make(map[string]*options), + instancesOptions: make(map[instanceID]*options), + runnersOptions: make(map[string]*options), + } + b.scriptGroups[id] = group + } + + return group +} + +func (b *batcher) build(ctx context.Context) (*Package, error) { + b.mu.Lock() + defer b.mu.Unlock() + + // Use the unexported calculateStaleVersion + if b.prevBuild != nil { + staleVersion := b.prevBuild.calculateStaleVersion() + if staleVersion == 0 { + return b.prevBuild, nil + } + } + + p, err := b.doBuild(ctx) + if err != nil { + return nil, err + } + b.prevBuild = p + return p, nil +} + +func (b *batcher) compile() error { + var err error + b.config, err = DecodeExternalOptions(b.configOptions.commit().opts) + if err != nil { + return err + } + + for _, v := range b.scriptGroups { + if err := v.compile(); err != nil { + return err + } + } + return nil +} + +func (b *batcher) doBuild(ctx context.Context) (*Package, error) { + keyPath := b.id + + type importContext struct { + name string + resourceGetter resource.ResourceGetter + scriptOptions *ScriptOptions // TODO1 remove resourceGetter? + } + + state := struct { + importResource *maps.Cache[string, resource.Resource] + resultResource *maps.Cache[string, resource.Resource] + importerImportContext *maps.Cache[string, importContext] + pathGroup *maps.Cache[string, string] + }{ + importResource: maps.NewCache[string, resource.Resource](), + resultResource: maps.NewCache[string, resource.Resource](), + importerImportContext: maps.NewCache[string, importContext](), + pathGroup: maps.NewCache[string, string](), + } + + // Entry points passed to ESBuid. + var entryPoints []string + addResource := func(group, pth string, r resource.Resource, isResult bool) { + state.pathGroup.Set(pth, group) + state.importResource.Set(pth, r) + if isResult { + state.resultResource.Set(pth, r) + } + entryPoints = append(entryPoints, pth) + } + + if err := b.compile(); err != nil { + return nil, err + } + + for k, v := range b.scriptGroups { + keyPath := keyPath + "_" + k + var runners []scriptRunnerTemplateContext + for _, vv := range v.runners.Sorted() { + runnerKeyPath := keyPath + "_" + vv.ID + runnerImpPath := paths.AddLeadingSlash(runnerKeyPath + "_runner" + vv.Resource.MediaType().FirstSuffix.FullSuffix) + runners = append(runners, scriptRunnerTemplateContext{script: vv, Import: runnerImpPath}) + addResource(k, runnerImpPath, vv.Resource, false) + } + + t := &batchTemplateContext{ + keyPath: keyPath, + ID: v.id, + Runners: runners, + } + + instances := v.instances.Sorted() + + for _, vv := range v.scripts.Sorted() { + keyPath := keyPath + "_" + vv.ID + opts := vv.ScriptOptions + impPath := path.Join(PrefixHugoVirtual, opts.Dir(), keyPath+opts.Resource.MediaType().FirstSuffix.FullSuffix) + impCtx := opts.ImportContext + + state.importerImportContext.Set(impPath, importContext{ + name: keyPath, + resourceGetter: impCtx, + scriptOptions: opts, + }) + + bt := scriptBatchTemplateContext{ + script: vv, + Import: impPath, + } + + state.importResource.Set(bt.Import, vv.Resource) + for _, vvv := range instances.ByScriptID(vv.ID) { + bt.Instances = append(bt.Instances, scriptInstanceBatchTemplateContext{instance: vvv}) + } + + t.Scripts = append(t.Scripts, bt) + } + + r, s, err := b.client.buildBatch(ctx, t) + if err != nil { + return nil, fmt.Errorf("failed to build batch: %w", err) + } + + state.importerImportContext.Set(s, importContext{ + name: s, + resourceGetter: nil, + scriptOptions: nil, + }) + + addResource(v.id, s, r, true) + } + + absPublishDir := b.client.d.AbsPublishDir + mediaTypes := b.client.d.ResourceSpec.MediaTypes() + cssMt, _, _ := mediaTypes.GetFirstBySuffix("css") + + outDir, err := b.client.d.MkdirTemp("hugo-jsbatch") + if err != nil { + return nil, err + } + + externalOptions := b.config + if externalOptions.Format == "" { + externalOptions.Format = "esm" + } + if externalOptions.Format != "esm" { + return nil, fmt.Errorf("only esm format is currently supported") + } + + jsOpts := Options{ + ExternalOptions: externalOptions, + InternalOptions: InternalOptions{ + DependencyManager: b.dependencyManager, + OutDir: outDir, + Write: true, + AllowOverwrite: true, + Splitting: true, + ImportOnResolveFunc: func(depsManager identity.Manager, imp string, args api.OnResolveArgs) string { + if _, found := state.importResource.Get(imp); found { + return imp + } + var importContextPath string + if args.Kind == api.ResolveEntryPoint { + importContextPath = args.Path + } else { + importContextPath = args.Importer + } + importContext, _ := state.importerImportContext.Get(importContextPath) + + if importContext.resourceGetter != nil { + resolved := importContext.resourceGetter.Get(imp) + if resolved != nil { + depsManager.AddIdentity(identity.FirstIdentity(resolved)) + imp := PrefixHugoVirtual + resolved.(resource.PathProvider).Path() + state.importResource.Set(imp, resolved) + state.importerImportContext.Set(imp, importContext) + return imp + + } + } + return "" + }, + ImportOnLoadFunc: func(args api.OnLoadArgs) string { + imp := args.Path + + if r, found := state.importResource.Get(imp); found { + content, err := r.(resource.ContentProvider).Content(context.Background()) // TODO1 + if err != nil { + panic(err) + } + return cast.ToString(content) + } + return "" + }, + ImportParamsOnLoadFunc: func(args api.OnLoadArgs) json.RawMessage { + if importContext, found := state.importerImportContext.Get(args.Path); found { + if importContext.scriptOptions != nil { + return importContext.scriptOptions.Params + } + } + return nil + }, + ErrorMessageResolveFunc: func(args api.Message) *ErrorMessageResolved { + if loc := args.Location; loc != nil { + path := strings.TrimPrefix(loc.File, NsHugoImportResolveFunc+":") + if r, found := state.importResource.Get(path); found { + path = strings.TrimPrefix(path, PrefixHugoVirtual) + var contentr hugio.ReadSeekCloser + if cp, ok := r.(hugio.ReadSeekCloserProvider); ok { + contentr, _ = cp.ReadSeekCloser() + } + return &ErrorMessageResolved{ + Content: contentr, + Path: path, + Message: args.Text, + } + + } + + } + return nil + }, + EntryPoints: entryPoints, + }, + } + + result, err := b.client.buildClient.Build(jsOpts) + if err != nil { + return nil, fmt.Errorf("failed to build bundle: %w", err) + } + + m := fromJSONToESBuildResultMeta(b.client.d.Conf.WorkingDir(), result.Metafile) + + groups := make(map[string]resource.Resources) + + createAndAddResource := func(filename, targetPath, group string, mt media.Type) error { + rd := resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return os.Open(filename) + }, + MediaType: mt, + TargetPath: targetPath, + // DependencyManager: idm, TODO1 + } + r, err := b.client.d.ResourceSpec.NewResource(rd) + if err != nil { + return err + } + + groups[group] = append(groups[group], r) + + return nil + } + + createAndAddResources := func(o esBuildResultMetaOutput) (bool, error) { + p := filepath.ToSlash(strings.TrimPrefix(o.filename, outDir)) + ext := path.Ext(p) + mt, _, found := mediaTypes.GetBySuffix(ext) + if !found { + return false, nil + } + groupPath := p + group, found := state.pathGroup.Get(groupPath) + + if !found { + return false, nil + } + + if err := createAndAddResource(o.filename, p, group, mt); err != nil { + return false, err + } + + if o.CSSBundle != "" { + p := filepath.ToSlash(strings.TrimPrefix(o.CSSBundle, outDir)) + if err := createAndAddResource(o.CSSBundle, p, group, cssMt); err != nil { + return false, err + } + } + + return true, nil + } + + for _, o := range m.Outputs { + handled, err := createAndAddResources(o) + if err != nil { + return nil, err + } + if !handled { + // Copy to destination. + p := strings.TrimPrefix(o.filename, outDir) + if err := hugio.CopyFile(hugofs.Os, o.filename, filepath.Join(absPublishDir, p)); err != nil { + return nil, fmt.Errorf("failed to copy %q to %q: %w", o.filename, absPublishDir, err) + } + } + } + + return &Package{ + origin: b, + outDir: outDir, + b: b, + id: path.Join(NsBatch, b.id), + Groups: groups, + }, nil +} + +func (b *batcher) reset() { + b.mu.Lock() + defer b.mu.Unlock() + for _, v := range b.scriptGroups { + // TODO1 check if this is complete. + v.Reset() + } +} + +// https://esbuild.github.io/api/#metafile +type esBuildResultMeta struct { + Outputs map[string]esBuildResultMetaOutput + + // Compiled values. + cssBundleEntryPoint map[string]esBuildResultMetaOutput +} + +func (e *esBuildResultMeta) Compile(cwd string) error { + // Rewrite the paths to be absolute. + // See https://github.com/evanw/esbuild/issues/338 + outputs := make(map[string]esBuildResultMetaOutput) + for k, v := range e.Outputs { + filename := filepath.Join(cwd, k) + if err := v.Compile(filename); err != nil { + return err + } + + if v.CSSBundle != "" { + v.CSSBundle = filepath.Join(cwd, v.CSSBundle) + } + outputs[filename] = v + } + e.Outputs = outputs + + e.cssBundleEntryPoint = make(map[string]esBuildResultMetaOutput) + for _, v := range e.Outputs { + if v.CSSBundle != "" { + e.cssBundleEntryPoint[v.CSSBundle] = v + } + } + return nil +} + +type esBuildResultMetaOutput struct { + Bytes int64 + Exports []string + Imports []esBuildResultMetaOutputImport + EntryPoint string + CSSBundle string + + // compiled values. + filename string +} + +func (e *esBuildResultMetaOutput) Compile(filename string) error { + e.filename = filename + return nil +} + +type esBuildResultMetaOutputImport struct { + Path string + Kind string +} + +// TODO1 remove the now superflous Close harness in the template package. + +type getOnce[T any] struct { + v T + once lazy.OnceMore +} + +type instance struct { + instanceID + *ParamsOptions +} + +type instanceID struct { + scriptID string + instanceID string +} + +type ( + instanceMap map[instanceID]*ParamsOptions + instances []*instance +) + +func (p instanceMap) Sorted() instances { + var a []*instance + for k, v := range p { + a = append(a, &instance{instanceID: k, ParamsOptions: v}) + } + sort.Slice(a, func(i, j int) bool { + ai := a[i] + aj := a[j] + if ai.instanceID.scriptID != aj.instanceID.scriptID { + return ai.instanceID.scriptID < aj.instanceID.scriptID + } + return ai.instanceID.instanceID < aj.instanceID.instanceID + }) + return a +} + +func (i instances) ByScriptID(id string) instances { + var a instances + for _, v := range i { + if v.instanceID.scriptID == id { + a = append(a, v) + } + } + return a +} + +type options struct { + getOnce[*optionsSetter] +} + +func (o *options) Reset() { + mu := o.once.ResetWithLock() + o.v.staleVersion.Store(0) + mu.Unlock() +} + +func (o *options) StaleVersion() uint32 { + return o.v.staleVersion.Load() +} + +type optionsGetSet struct { + Resource resource.Resource + ImportContext any + Params map[string]any + + // Compiled values + compiled *ScriptOptions +} + +func (s *optionsGetSet) GetOptions() *ScriptOptions { + return s.compiled +} + +func (s *optionsGetSet) SetOptions(m map[string]any) string { + if err := mapstructure.WeakDecode(m, &s); err != nil { + panic(err) + } + + paramsJSON, err := json.Marshal(s.Params) + if err != nil { + panic(err) + } + + s.compiled = &ScriptOptions{ + Resource: s.Resource, + ImportContext: resource.NewResourceGetter(s.ImportContext), + Params: paramsJSON, + } + + return "" +} + +type optionsSetter struct { + staleVersion atomic.Uint32 + opts map[string]any +} + +// TODO1 try to avoid stale page resources when changing the head. +func (o *optionsSetter) SetOptions(m map[string]any) string { + if o.opts != nil { + if reflect.DeepEqual(o.opts, m) { + return "" + } + var isStale bool + for k, v := range m { + vv, found := o.opts[k] + if !found { + isStale = true + } else { + if si, ok := vv.(resource.StaleInfo); ok { + isStale = si.StaleVersion() > 0 + } else { + isStale = !reflect.DeepEqual(v, vv) + } + } + + if isStale { + break + } + } + + if !isStale { + return "" + } + + o.staleVersion.Add(1) + } + + o.opts = m + + return "" +} + +type script struct { + ID string + *ScriptOptions +} + +type scriptBatchTemplateContext struct { + *script + Import string + Instances []scriptInstanceBatchTemplateContext +} + +func (c scriptBatchTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + Instances []scriptInstanceBatchTemplateContext `json:"instances"` + }{ + ID: c.ID, + Instances: c.Instances, + }) +} + +func (b scriptBatchTemplateContext) RunnerJSON(i int) string { + script := fmt.Sprintf("Script%d", i) + + v := struct { + ID string `json:"id"` + + // Read-only live JavaScript binding. + Binding string `json:"binding"` + Instances []scriptInstanceBatchTemplateContext `json:"instances"` + }{ + b.ID, + script, + b.Instances, + } + + bb, err := json.Marshal(v) + if err != nil { + panic(err) + } + s := string(bb) + + // Remove the quotes to make it a valid JS object. + s = strings.ReplaceAll(s, fmt.Sprintf("%q", script), script) + + return s +} + +type scriptGroup struct { + mu sync.Mutex + + id string + + client *BatcherClient + + scriptsOptions map[string]*options + instancesOptions map[instanceID]*options + runnersOptions map[string]*options + + // Compiled. + scripts scriptMap + instances instanceMap + runners scriptMap +} + +func (s *scriptGroup) Instance(sid, iid string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + id := instanceID{scriptID: sid, instanceID: iid} + if v, found := s.instancesOptions[id]; found { + return v.Get() + } + s.instancesOptions[id] = newOptions() + return s.instancesOptions[id].Get() +} + +func (g *scriptGroup) Reset() { + for _, v := range g.scriptsOptions { + v.Reset() + } + for _, v := range g.instancesOptions { + v.Reset() + } + for _, v := range g.runnersOptions { + v.Reset() + } +} + +func (s *scriptGroup) Runner(id string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + if v, found := s.runnersOptions[id]; found { + return v.Get() + } + s.runnersOptions[id] = newOptions() + return s.runnersOptions[id].Get() +} + +func (s *scriptGroup) Script(id string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + if v, found := s.scriptsOptions[id]; found { + return v.Get() + } + s.scriptsOptions[id] = newOptions() + return s.scriptsOptions[id].Get() +} + +func (s *scriptGroup) errFailedToCompile(what, id string, err error) error { + return fmt.Errorf("failed to compile %s for %q > %q: %w", what, s.id, id, err) +} + +func (s *scriptGroup) compile() error { + // TODO1 lock? + s.scripts = make(map[string]*ScriptOptions) + s.instances = make(map[instanceID]*ParamsOptions) + s.runners = make(map[string]*ScriptOptions) + + for k, v := range s.scriptsOptions { + compiled, err := compileScriptOptions(v, "*") + if err != nil { + return s.errFailedToCompile("Script", k, err) + } + s.scripts[k] = compiled + } + + for k, v := range s.instancesOptions { + compiled, err := compileParamsOptions(v) + if err != nil { + return err + } + s.instances[k] = compiled + } + + for k, v := range s.runnersOptions { + compiled, err := compileScriptOptions(v, "default") + if err != nil { + return s.errFailedToCompile("Runner", k, err) + } + s.runners[k] = compiled + } + + return nil +} + +type scriptGroups map[string]*scriptGroup + +func (s scriptGroups) Sorted() []*scriptGroup { + var a []*scriptGroup + for _, v := range s { + a = append(a, v) + } + sort.Slice(a, func(i, j int) bool { + return a[i].id < a[j].id + }) + return a +} + +type scriptInstanceBatchTemplateContext struct { + *instance +} + +func (c scriptInstanceBatchTemplateContext) ID() string { + return c.instanceID.instanceID +} + +func (c scriptInstanceBatchTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + Params json.RawMessage `json:"params"` + }{ + ID: c.instanceID.instanceID, + Params: c.Params, + }) +} + +type scriptMap map[string]*ScriptOptions + +func (s scriptMap) Sorted() []*script { + var a []*script + for k, v := range s { + a = append(a, &script{ID: k, ScriptOptions: v}) + } + sort.Slice(a, func(i, j int) bool { + return a[i].ID < a[j].ID + }) + return a +} + +type scriptRunnerTemplateContext struct { + *script + Import string +} + +func (c scriptRunnerTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + }{ + ID: c.ID, + }) +} + +func (g *getOnce[T]) commit() T { + g.once.Do(func() {}) + return g.v +} + +func compileParamsOptions(o *options) (*ParamsOptions, error) { + v := struct { + Params map[string]any + }{} + + m := o.commit().opts + + if err := mapstructure.WeakDecode(m, &v); err != nil { + return nil, err + } + + paramsJSON, err := json.Marshal(v.Params) + if err != nil { + return nil, err + } + + return &ParamsOptions{ + Params: paramsJSON, + }, nil +} + +func compileScriptOptions(o *options, defaultExport string) (*ScriptOptions, error) { + v := struct { + Resource resource.Resource + ImportContext any + Export string + Params map[string]any + }{} + + m := o.commit().opts + + if err := mapstructure.WeakDecode(m, &v); err != nil { + panic(err) + } + + var paramsJSON []byte + if v.Params != nil { + var err error + paramsJSON, err = json.Marshal(v.Params) + if err != nil { + panic(err) + } + } + + if v.Export == "" { + v.Export = defaultExport + } + + compiled := &ScriptOptions{ + Resource: v.Resource, + Export: v.Export, + ImportContext: resource.NewResourceGetter(v.ImportContext), + Params: paramsJSON, + } + + if compiled.Resource == nil { + return nil, fmt.Errorf("resource not set") + } + + return compiled, nil +} + +// TODO1 remove. +func deb(what string, v ...any) { + fmt.Println(what, v) +} + +func fromJSONToESBuildResultMeta(cwd, jsons string) esBuildResultMeta { + var m esBuildResultMeta + if err := json.Unmarshal([]byte(jsons), &m); err != nil { + panic(err) + } + if err := m.Compile(cwd); err != nil { + panic(err) + } + return m +} + +func logTime(name string, start time.Time) { + elapsed := time.Since(start) + fmt.Printf("%s in %s\n", name, elapsed) +} diff --git a/internal/js/esbuild/batch_integration_test.go b/internal/js/esbuild/batch_integration_test.go new file mode 100644 index 00000000000..fe35e1ae98c --- /dev/null +++ b/internal/js/esbuild/batch_integration_test.go @@ -0,0 +1,428 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package esbuild_test + +import ( + "fmt" + "strings" + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/bep/logg" + "github.com/gohugoio/hugo/hugolib" +) + +// Used to test misc. error situations etc. +const jsBatchFilesTemplate = ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "section", "page"] +-- assets/js/styles.css -- +body { + background-color: red; +} +-- assets/js/main.js -- +import './styles.css'; +console.log("Hello, Main!"); +-- assets/js/runner.js -- +console.log("Hello, Runner!"); +-- layouts/index.html -- +{{ $batch := (js.Batch "mybatch" site.Home.Store) }} +{{ with $batch.Group "mygroup" }} + {{ with .Runner "run" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script "main" }} + {{ .SetOptions (dict "resource" (resources.Get "js/main.js")) }} + {{ end }} +{{ end }} + +{{ with (templates.Defer (dict "key" "global")) }} + Defer: + {{ $batch := (js.Batch "mybatch" site.Home.Store) }} + {{ range $k, $v := $batch.Build.Groups }} + {{ $k }}: + {{ range . }} + RelPermalink: {{ .RelPermalink }} + Content: {{ .Content | safeHTML }} + {{ end }} + {{ end }} +{{ end }} +` + +// Just to verify that the above file setup works. +func TestBatchTemplateOKBuild(t *testing.T) { + b := hugolib.Test(t, jsBatchFilesTemplate, hugolib.TestOptWithOSFs()) + b.AssertPublishDir("mybatch_mygroup.js", "mybatch_mygroup.css") +} + +func TestBatchErrorScriptResourceNotSet(t *testing.T) { + files := strings.Replace(jsBatchFilesTemplate, `(resources.Get "js/main.js")`, `(resources.Get "js/doesnotexist.js")`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `failed to build Batch "mybatch": failed to compile Script for "mygroup" > "main": resource not set`) +} + +func TestBatchErrorRunnerResourceNotSet(t *testing.T) { + files := strings.Replace(jsBatchFilesTemplate, `(resources.Get "js/runner.js")`, `(resources.Get "js/doesnotexist.js")`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `failed to build Batch "mybatch": failed to compile Runner for "mygroup" > "run": resource not set`) +} + +func TestBatchErrorScriptResourceSyntaxError(t *testing.T) { + files := strings.Replace(jsBatchFilesTemplate, `console.log("Hello, Main!");`, `console.flog("Hello, Main!"`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `failed to build Batch "mybatch": failed to build bundle: "/js/mybatch_mygroup_main.js:2:27": Expected ")" but found end of file`) +} + +func TestBatch(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term"] +disableLiveReload = true +baseURL = "https://example.com" +-- package.json -- +{ + "devDependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} +-- assets/js/shims/react.js -- +-- assets/js/shims/react-dom.js -- +module.exports = window.ReactDOM; +module.exports = window.React; +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/mybundlestyles.css -- +@import './foo.css'; +@import './bar.css'; +@import './otherbundlestyles.css'; + +.mybundlestyles { + background-color: blue; +} +-- content/mybundle/bundlereact.jsx -- +import * as React from "react"; +import './foo.css'; +import './mybundlestyles.css'; +window.React1 = React; + +let text = 'Click me, too!' + +export default function MyBundleButton() { + return ( + + ) +} + +-- assets/js/reactrunner.js -- +import * as ReactDOM from 'react-dom/client'; +import * as React from 'react'; + +export default function Run(modules) { + for (const module of modules) { + for (const instance of module.instances) { + /* This is a convention in this project. */ + let elId = §§${module.id}-${instance.id}§§; + let el = document.getElementById(elId); + if (!el) { + console.warn(§§Element with id ${elId} not found§§); + continue; + } + const root = ReactDOM.createRoot(el); + const reactEl = React.createElement(module.mod, instance.params); + root.render(reactEl); + } + } +} +-- assets/other/otherbundlestyles.css -- +.otherbundlestyles { + background-color: red; +} +-- assets/other/foo.css -- +@import './bar.css'; + +.foo { + background-color: blue; +} +-- assets/other/bar.css -- +.bar { + background-color: red; +} +-- assets/js/button.css -- +button { + background-color: red; +} +-- assets/js/bar.css -- +.bar-assets { + background-color: red; +} +-- assets/js/helper.js -- +import './bar.css' + +export function helper() { + console.log('helper'); +} + +-- assets/js/react1styles_nested.css -- +.react1styles_nested { + background-color: red; +} +-- assets/js/react1styles.css -- +@import './react1styles_nested.css'; +.react1styles { + background-color: red; +} +-- assets/js/react1.jsx -- +import * as React from "react"; +import './button.css' +import './foo.css' +import './react1styles.css' + +window.React1 = React; + +let text = 'Click me' + +export default function MyButton() { + return ( + + ) +} + +-- assets/js/react2.jsx -- +import * as React from "react"; +import { helper } from './helper.js' +import './foo.css' + +window.React2 = React; + +let text = 'Click me, too!' + +export function MyOtherButton() { + return ( + + ) +} +-- assets/js/main1.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main1.React', React) +console.log('main1.params.id', params.id) + +-- assets/js/main2.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main2.React', React) +console.log('main2.params.id', params.id) + +export default function Main2() {}; + +-- assets/js/main3.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main3.params.id', params.id) + +export default function Main3() {}; + +-- layouts/_default/single.html -- +Single. +{{ $r := .Resources.GetMatch "*.jsx" }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} + {{ with $batch.Config }} + {{ $shims := dict "react" "js/shims/react.js" "react-dom/client" "js/shims/react-dom.js" }} + {{ .SetOptions (dict + "target" "es2018" + "params" (dict "id" "config") + "shims" $shims + ) + }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Script "r3" }} + {{ .SetOptions (dict + "resource" $r + "importContext" (slice $ $otherCSS) + "params" (dict "id" "r3") + ) + }} + {{ end }} + {{ with .Instance "r3" "r2i1" }} + {{ .SetOptions (dict "title" "r2 instance 1")}} + {{ end }} +{{ end }} +-- layouts/index.html -- +Home. +{{ with (templates.Defer (dict "key" "global")) }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ range $k, $v := $batch.Build.Groups }} + {{ $k }}: + {{ range . }} + {{ .RelPermalink }} + {{ end }} + {{ end }} +{{ end }} +{{ $myContentBundle := site.GetPage "mybundle" }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} +{{ with $batch.Group "mains" }} + {{ with .Script "main1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main1.js") + "params" (dict "id" "main1") + ) + }} + {{ end }} + {{ with .Script "main2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main2.js") + "params" (dict "id" "main2") + ) + }} + {{ end }} + {{ with .Script "main3" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main3.js") + ) + }} + {{ end }} +{{ with .Instance "main1" "m1i1" }}{{ .SetOptions (dict "params" (dict "title" "Main1 Instance 1"))}}{{ end }} +{{ with .Instance "main1" "m1i2" }}{{ .SetOptions (dict "params" (dict "title" "Main1 Instance 2"))}}{{ end }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Runner "reactrunner" }} + {{ .SetOptions ( dict "resource" (resources.Get "js/reactrunner.js") )}} + {{ end }} + {{ with .Script "r1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react1.jsx") + "importContext" (slice $myContentBundle $otherCSS) + "params" (dict "id" "r1") + ) + }} + {{ end }} + {{ with .Instance "r1" "i1" }}{{ .SetOptions (dict "params" (dict "title" "Instance 1"))}}{{ end }} + {{ with .Instance "r1" "i2" }}{{ .SetOptions (dict "params" (dict "title" "Instance 2"))}}{{ end }} + {{ with .Script "r2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react2.jsx") + "export" "MyOtherButton" + "importContext" $otherCSS + "params" (dict "id" "r2") + ) + }} + {{ end }} + {{ with .Instance "r2" "i1" }}{{ .SetOptions (dict "params" (dict "title" "Instance 2-1"))}}{{ end }} +{{ end }} + +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + NeedsOsFS: true, + NeedsNpmInstall: true, + TxtarString: files, + Running: true, + LogLevel: logg.LevelWarn, + // PrintAndKeepTempDir: true, + }).Build() + + fmt.Println(b.LogString()) + + b.AssertFileContent("public/mybundle_reactbatch.css", + ".bar {", + ) + + // Verify params resolution. + b.AssertFileContent("public/mybundle_mains.js", + ` +var id = "main1"; +console.log("main1.params.id", id); +var id2 = "main2"; +console.log("main2.params.id", id2); + +# Params from top level config. +var id3 = "config"; +console.log("main3.params.id", id3); +`) + + b.EditFileReplaceAll("content/mybundle/mybundlestyles.css", ".mybundlestyles", ".mybundlestyles-edit").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".mybundlestyles-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar {", ".bar-edit {").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".bar-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar-edit {", ".bar-edit2 {").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".bar-edit2 {") +} + +// TODO1 move this. +func TestResourcesGet(t *testing.T) { + files := ` +-- hugo.toml -- +-- assets/text/txt1.txt -- +Text 1. +-- assets/text/txt2.txt -- +Text 2. +-- assets/text/sub/txt3.txt -- +Text 3. +-- assets/text/sub/txt4.txt -- +Text 4. +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/txt1.txt -- +Text 1. +-- content/mybundle/sub/txt2.txt -- +Text 1. +-- layouts/index.html -- +{{ $mybundle := site.GetPage "mybundle" }} +{{ $subResources := resources.Match "/text/sub/*.*" }} +{{ $subResourcesMount := $subResources.Mount "/text/sub" "/newroot" }} +resources:text/txt1.txt:{{ with resources.Get "text/txt1.txt" }}{{ .Name }}{{ end }}| +resources:text/txt2.txt:{{ with resources.Get "text/txt2.txt" }}{{ .Name }}{{ end }}| +resources:text/sub/txt3.txt:{{ with resources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResources.range:{{ range $subResources }}{{ .Name }}|{{ end }}| +subResources:"text/sub/txt3.txt:{{ with $subResources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResourcesMount:/newroot/txt3.txt:{{ with $subResourcesMount.Get "/newroot/txt3.txt" }}{{ .Name }}{{ end }}| +page:txt1.txt:{{ with $mybundle.Resources.Get "txt1.txt" }}{{ .Name }}{{ end }}| +page:./txt1.txt:{{ with $mybundle.Resources.Get "./txt1.txt" }}{{ .Name }}{{ end }}| +page:sub/txt2.txt:{{ with $mybundle.Resources.Get "sub/txt2.txt" }}{{ .Name }}{{ end }}| +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/index.html", ` +resources:text/txt1.txt:/text/txt1.txt| +resources:text/txt2.txt:/text/txt2.txt| +resources:text/sub/txt3.txt:/text/sub/txt3.txt| +subResources:"text/sub/txt3.txt:/text/sub/txt3.txt| +subResourcesMount:/newroot/txt3.txt:/text/sub/txt3.txt| +page:txt1.txt:txt1.txt| +page:./txt1.txt:txt1.txt| +page:sub/txt2.txt:sub/txt2.txt| +`) +} + +// TODO1 check .Name in bundles on renames. diff --git a/internal/js/esbuild/build.go b/internal/js/esbuild/build.go new file mode 100644 index 00000000000..c3c6e3b98b5 --- /dev/null +++ b/internal/js/esbuild/build.go @@ -0,0 +1,181 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package esbuild provides functions for building JavaScript resources. +package esbuild + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources" +) + +// NewBuildClient creates a new BuildClient. +func NewBuildClient(fs *filesystems.SourceFilesystem, rs *resources.Spec) *BuildClient { + return &BuildClient{ + rs: rs, + sfs: fs, + } +} + +type BuildClient struct { + rs *resources.Spec + sfs *filesystems.SourceFilesystem +} + +func (c *BuildClient) Build(opts Options) (api.BuildResult, error) { + dependencyManager := opts.DependencyManager + if dependencyManager == nil { + dependencyManager = identity.NopManager + } + + opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") + + if err := opts.validate(); err != nil { + return api.BuildResult{}, err + } + + buildOptions, err := toBuildOptions(opts) + if err != nil { + return api.BuildResult{}, err + } + + buildOptions.Plugins, err = createBuildPlugins(c.rs, dependencyManager, opts) + if err != nil { + return api.BuildResult{}, err + } + + if buildOptions.Sourcemap == api.SourceMapExternal && buildOptions.Outdir == "" { + buildOptions.Outdir, err = os.MkdirTemp(os.TempDir(), "compileOutput") + if err != nil { + return api.BuildResult{}, err + } + defer os.Remove(buildOptions.Outdir) + } + + if opts.Inject != nil { + // Resolve the absolute filenames. + for i, ext := range opts.Inject { + impPath := filepath.FromSlash(ext) + if filepath.IsAbs(impPath) { + return api.BuildResult{}, fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") + } + + m := resolveComponentInAssets(c.rs.Assets.Fs, impPath) + + if m == nil { + return api.BuildResult{}, fmt.Errorf("inject: file %q not found", ext) + } + + opts.Inject[i] = m.Filename + + } + + buildOptions.Inject = opts.Inject + + } + + result := api.Build(buildOptions) + + if len(result.Errors) > 0 { + createErr := func(msg api.Message) error { + if msg.Location == nil { + return errors.New(msg.Text) + } + var ( + contentr hugio.ReadSeekCloser + errorMessage string + loc = msg.Location + errorPath = loc.File + err error + ) + + var resolvedError *ErrorMessageResolved + + if opts.ErrorMessageResolveFunc != nil { + resolvedError = opts.ErrorMessageResolveFunc(msg) + } + + if resolvedError == nil { + if errorPath == stdinImporter { + errorPath = "TODO1" // transformCtx.SourcePath + } + + errorMessage = msg.Text + // TODO1 handle all namespaces, make more general + errorMessage = strings.ReplaceAll(errorMessage, NsHugoImport+":", "") + + if strings.HasPrefix(errorPath, NsHugoImport) { + errorPath = strings.TrimPrefix(errorPath, NsHugoImport+":") + contentr, err = hugofs.Os.Open(errorPath) + } else { + var fi os.FileInfo + fi, err = c.sfs.Fs.Stat(errorPath) + if err == nil { + m := fi.(hugofs.FileMetaInfo).Meta() + errorPath = m.Filename + contentr, err = m.Open() + } + } + } else { + contentr = resolvedError.Content + errorPath = resolvedError.Path + errorMessage = resolvedError.Message + } + + if contentr != nil { + defer contentr.Close() + } + + if err == nil { + fe := herrors. + NewFileErrorFromName(errors.New(errorMessage), errorPath). + UpdatePosition(text.Position{Offset: -1, LineNumber: loc.Line, ColumnNumber: loc.Column}). + UpdateContent(contentr, nil) + + return fe + } + + return fmt.Errorf("%s", errorMessage) + } + + var errors []error + + for _, msg := range result.Errors { + errors = append(errors, createErr(msg)) + } + + // Return 1, log the rest. + for i, err := range errors { + if i > 0 { + c.rs.Logger.Errorf("js.Build failed: %s", err) + } + } + + return result, errors[0] + } + + return result, nil +} diff --git a/internal/js/esbuild/options.go b/internal/js/esbuild/options.go new file mode 100644 index 00000000000..82d8528efa5 --- /dev/null +++ b/internal/js/esbuild/options.go @@ -0,0 +1,301 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/identity" + + "github.com/evanw/esbuild/pkg/api" + + "github.com/gohugoio/hugo/media" + "github.com/mitchellh/mapstructure" +) + +type Options struct { + ExternalOptions + InternalOptions +} + +func (opts *Options) validate() error { + if opts.ImportOnResolveFunc != nil && opts.ImportOnLoadFunc == nil { + return fmt.Errorf("ImportOnLoadFunc must be set if ImportOnResolveFunc is set") + } + if opts.ImportOnResolveFunc == nil && opts.ImportOnLoadFunc != nil { + return fmt.Errorf("ImportOnResolveFunc must be set if ImportOnLoadFunc is set") + } + return nil +} + +// InternalOptions holds internal options for the js.Build template function. +type InternalOptions struct { + MediaType media.Type + OutDir string + Contents string + SourceDir string + ResolveDir string + + DependencyManager identity.Manager + + Write bool // Set to false to write to memory. + AllowOverwrite bool + Splitting bool + TsConfig string + EntryPoints []string + ImportOnResolveFunc func(identity.Manager, string, api.OnResolveArgs) string + ImportOnLoadFunc func(api.OnLoadArgs) string + ImportParamsOnLoadFunc func(args api.OnLoadArgs) json.RawMessage + ErrorMessageResolveFunc func(api.Message) *ErrorMessageResolved + Stdin bool +} + +type ErrorMessageResolved struct { + Path string + Message string + Content hugio.ReadSeekCloser +} + +// ExternalOptions holds user facing options for the js.Build template function. +type ExternalOptions struct { + // If not set, the source path will be used as the base target path. + // Note that the target path's extension may change if the target MIME type + // is different, e.g. when the source is TypeScript. + TargetPath string + + // Whether to minify to output. + Minify bool + + // Whether to write mapfiles + SourceMap string + + // The language target. + // One of: es2015, es2016, es2017, es2018, es2019, es2020 or esnext. + // Default is esnext. + Target string + + // The output format. + // One of: iife, cjs, esm + // Default is to esm. + Format string + + // External dependencies, e.g. "react". + Externals []string + + // This option allows you to automatically replace a global variable with an import from another file. + // The filenames must be relative to /assets. + // See https://esbuild.github.io/api/#inject + Inject []string + + // User defined symbols. + Defines map[string]any + + // Maps a component import to another. + Shims map[string]string + + // User defined params. Will be marshaled to JSON and available as "@params", e.g. + // import * as params from '@params'; + Params any + + // What to use instead of React.createElement. + JSXFactory string + + // What to use instead of React.Fragment. + JSXFragment string + + // What to do about JSX syntax. + // See https://esbuild.github.io/api/#jsx + JSX string + + // Which library to use to automatically import JSX helper functions from. Only works if JSX is set to automatic. + // See https://esbuild.github.io/api/#jsx-import-source + JSXImportSource string + + // There is/was a bug in WebKit with severe performance issue with the tracking + // of TDZ checks in JavaScriptCore. + // + // Enabling this flag removes the TDZ and `const` assignment checks and + // may improve performance of larger JS codebases until the WebKit fix + // is in widespread use. + // + // See https://bugs.webkit.org/show_bug.cgi?id=199866 + // Deprecated: This no longer have any effect and will be removed. + // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba + AvoidTDZ bool +} + +func DecodeExternalOptions(m map[string]any) (ExternalOptions, error) { + var opts ExternalOptions + + if err := mapstructure.WeakDecode(m, &opts); err != nil { + return opts, err + } + + if opts.TargetPath != "" { + opts.TargetPath = paths.ToSlashTrimLeading(opts.TargetPath) + } + + opts.Target = strings.ToLower(opts.Target) + opts.Format = strings.ToLower(opts.Format) + + return opts, nil +} + +func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { + var target api.Target + switch opts.Target { + case "", "esnext": + target = api.ESNext + case "es5": + target = api.ES5 + case "es6", "es2015": + target = api.ES2015 + case "es2016": + target = api.ES2016 + case "es2017": + target = api.ES2017 + case "es2018": + target = api.ES2018 + case "es2019": + target = api.ES2019 + case "es2020": + target = api.ES2020 + case "es2021": + target = api.ES2021 + case "es2022": + target = api.ES2022 + case "es2023": + target = api.ES2023 + default: + err = fmt.Errorf("invalid target: %q", opts.Target) + return + } + + mediaType := opts.MediaType + if mediaType.IsZero() { + mediaType = media.Builtin.JavascriptType + } + + var loader api.Loader + switch mediaType.SubType { + // TODO(bep) ESBuild support a set of other loaders, but I currently fail + // to see the relevance. That may change as we start using this. + case media.Builtin.JavascriptType.SubType: + loader = api.LoaderJS + case media.Builtin.TypeScriptType.SubType: + loader = api.LoaderTS + case media.Builtin.TSXType.SubType: + loader = api.LoaderTSX + case media.Builtin.JSXType.SubType: + loader = api.LoaderJSX + default: + err = fmt.Errorf("unsupported Media Type: %q", opts.MediaType) + return + } + + var format api.Format + // One of: iife, cjs, esm + switch opts.Format { + case "", "iife": + format = api.FormatIIFE + case "esm": + format = api.FormatESModule + case "cjs": + format = api.FormatCommonJS + default: + err = fmt.Errorf("unsupported script output format: %q", opts.Format) + return + } + + var jsx api.JSX + switch opts.JSX { + case "", "transform": + jsx = api.JSXTransform + case "preserve": + jsx = api.JSXPreserve + case "automatic": + jsx = api.JSXAutomatic + default: + err = fmt.Errorf("unsupported jsx type: %q", opts.JSX) + return + } + + var defines map[string]string + if opts.Defines != nil { + defines = maps.ToStringMapString(opts.Defines) + } + + // By default we only need to specify outDir and no outFile + outDir := opts.OutDir + outFile := "" + var sourceMap api.SourceMap + switch opts.SourceMap { + case "inline": + sourceMap = api.SourceMapInline + case "external": + sourceMap = api.SourceMapExternal + case "": + sourceMap = api.SourceMapNone + default: + err = fmt.Errorf("unsupported sourcemap type: %q", opts.SourceMap) + return + } + + buildOptions = api.BuildOptions{ + Outfile: outFile, + Bundle: true, + Metafile: true, // TODO1 batch only. + + Target: target, + Format: format, + Sourcemap: sourceMap, + + MinifyWhitespace: opts.Minify, + MinifyIdentifiers: opts.Minify, + MinifySyntax: opts.Minify, + + Outdir: outDir, + Write: opts.Write, + AllowOverwrite: opts.AllowOverwrite, + Splitting: opts.Splitting, + + Define: defines, + External: opts.Externals, + + JSXFactory: opts.JSXFactory, + JSXFragment: opts.JSXFragment, + + JSX: jsx, + JSXImportSource: opts.JSXImportSource, + + Tsconfig: opts.TsConfig, + + EntryPoints: opts.EntryPoints, + } + + if opts.Stdin { + // This makes ESBuild pass `stdin` as the Importer to the import. + buildOptions.Stdin = &api.StdinOptions{ + Contents: opts.Contents, + ResolveDir: opts.ResolveDir, + Loader: loader, + } + } + return +} diff --git a/internal/js/esbuild/options_test.go b/internal/js/esbuild/options_test.go new file mode 100644 index 00000000000..af024315277 --- /dev/null +++ b/internal/js/esbuild/options_test.go @@ -0,0 +1,220 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "testing" + + "github.com/gohugoio/hugo/media" + + "github.com/evanw/esbuild/pkg/api" + + qt "github.com/frankban/quicktest" +) + +func TestToBuildOptions(t *testing.T) { + c := qt.New(t) + + opts, err := toBuildOptions( + Options{ + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ESNext, + Format: api.FormatIIFE, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", + Format: "cjs", + Minify: true, + AvoidTDZ: true, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Sourcemap: api.SourceMapInline, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Sourcemap: api.SourceMapInline, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "external", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Sourcemap: api.SourceMapExternal, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + JSX: "automatic", JSXImportSource: "preact", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + }, + ) + + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Metafile: true, + Target: api.ESNext, + Format: api.FormatIIFE, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + JSX: api.JSXAutomatic, + JSXImportSource: "preact", + }) +} + +func TestToBuildOptionsTarget(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + target string + expect api.Target + }{ + {"es2015", api.ES2015}, + {"es2016", api.ES2016}, + {"es2017", api.ES2017}, + {"es2018", api.ES2018}, + {"es2019", api.ES2019}, + {"es2020", api.ES2020}, + {"es2021", api.ES2021}, + {"es2022", api.ES2022}, + {"es2023", api.ES2023}, + {"", api.ESNext}, + {"esnext", api.ESNext}, + } { + c.Run(test.target, func(c *qt.C) { + opts, err := toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: test.target, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + + c.Assert(err, qt.IsNil) + c.Assert(opts.Target, qt.Equals, test.expect) + }) + } +} diff --git a/internal/js/esbuild/resolve.go b/internal/js/esbuild/resolve.go new file mode 100644 index 00000000000..32fdb095e96 --- /dev/null +++ b/internal/js/esbuild/resolve.go @@ -0,0 +1,275 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources" + "github.com/spf13/afero" +) + +const ( + NsHugoImport = "ns-hugo-import" + NsHugoImportResolveFunc = "ns-hugo-import-resolvefunc" + nsHugoParams = "ns-hugo-params" + + stdinImporter = "" +) + +const ( + PrefixHugoVirtual = "@hugo-virtual" +) + +var extensionToLoaderMap = map[string]api.Loader{ + ".js": api.LoaderJS, + ".mjs": api.LoaderJS, + ".cjs": api.LoaderJS, + ".jsx": api.LoaderJSX, + ".ts": api.LoaderTS, + ".tsx": api.LoaderTSX, + ".css": api.LoaderCSS, + ".json": api.LoaderJSON, + ".txt": api.LoaderText, +} + +func loaderFromFilename(filename string) api.Loader { + l, found := extensionToLoaderMap[filepath.Ext(filename)] + if found { + return l + } + return api.LoaderJS +} + +func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { + findFirst := func(base string) *hugofs.FileMeta { + // This is the most common sub-set of ESBuild's default extensions. + // We assume that imports of JSON, CSS etc. will be using their full + // name with extension. + for _, ext := range []string{".js", ".ts", ".tsx", ".jsx"} { + if strings.HasSuffix(impPath, ext) { + // Import of foo.js.js need the full name. + continue + } + if fi, err := fs.Stat(base + ext); err == nil { + return fi.(hugofs.FileMetaInfo).Meta() + } + } + + // Not found. + return nil + } + + var m *hugofs.FileMeta + + // We need to check if this is a regular file imported without an extension. + // There may be ambiguous situations where both foo.js and foo/index.js exists. + // This import order is in line with both how Node and ESBuild's native + // import resolver works. + + // It may be a regular file imported without an extension, e.g. + // foo or foo/index. + m = findFirst(impPath) + if m != nil { + return m + } + + base := filepath.Base(impPath) + if base == "index" { + // try index.esm.js etc. + m = findFirst(impPath + ".esm") + if m != nil { + return m + } + } + + // Check the path as is. + fi, err := fs.Stat(impPath) + + if err == nil { + if fi.IsDir() { + m = findFirst(filepath.Join(impPath, "index")) + if m == nil { + m = findFirst(filepath.Join(impPath, "index.esm")) + } + } else { + m = fi.(hugofs.FileMetaInfo).Meta() + } + } else if strings.HasSuffix(base, ".js") { + m = findFirst(strings.TrimSuffix(impPath, ".js")) + } + + return m +} + +func createBuildPlugins(rs *resources.Spec, depsManager identity.Manager, opts Options) ([]api.Plugin, error) { + fs := rs.Assets + + resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { + impPath := args.Path + shimmed := false + if opts.Shims != nil { + override, found := opts.Shims[impPath] + if found { + impPath = override + shimmed = true + } + } + + if opts.ImportOnResolveFunc != nil { + if s := opts.ImportOnResolveFunc(depsManager, impPath, args); s != "" { + return api.OnResolveResult{Path: s, Namespace: NsHugoImportResolveFunc}, nil + } + } + + importer := args.Importer + + isStdin := importer == stdinImporter + var relDir string + if !isStdin { + if strings.HasPrefix(importer, PrefixHugoVirtual) { + relDir = filepath.Dir(strings.TrimPrefix(importer, PrefixHugoVirtual)) + } else { + rel, found := fs.MakePathRelative(importer, true) + + if !found { + if shimmed { + relDir = opts.SourceDir + } else { + // Not in any of the /assets folders. + // This is an import from a node_modules, let + // ESBuild resolve this. + return api.OnResolveResult{}, nil + } + } else { + relDir = filepath.Dir(rel) + } + } + } else { + relDir = opts.SourceDir + } + + // Imports not starting with a "." is assumed to live relative to /assets. + // Hugo makes no assumptions about the directory structure below /assets. + if relDir != "" && strings.HasPrefix(impPath, ".") { + impPath = filepath.Join(relDir, impPath) + } + + m := resolveComponentInAssets(fs.Fs, impPath) + + if m != nil { + depsManager.AddIdentity(m.PathInfo) + + // Store the source root so we can create a jsconfig.json + // to help IntelliSense when the build is done. + // This should be a small number of elements, and when + // in server mode, we may get stale entries on renames etc., + // but that shouldn't matter too much. + rs.JSConfigBuilder.AddSourceRoot(m.SourceRoot) + return api.OnResolveResult{Path: m.Filename, Namespace: NsHugoImport}, nil + } + + // Fall back to ESBuild's resolve. + return api.OnResolveResult{}, nil + } + + importResolver := api.Plugin{ + Name: "hugo-import-resolver", + Setup: func(build api.PluginBuild) { + build.OnResolve(api.OnResolveOptions{Filter: `.*`}, + func(args api.OnResolveArgs) (api.OnResolveResult, error) { + return resolveImport(args) + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: NsHugoImport}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + b, err := os.ReadFile(args.Path) + if err != nil { + return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + } + c := string(b) + + return api.OnLoadResult{ + // See https://github.com/evanw/esbuild/issues/502 + // This allows all modules to resolve dependencies + // in the main project's node_modules. + ResolveDir: opts.ResolveDir, + Contents: &c, + Loader: loaderFromFilename(args.Path), + }, nil + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: NsHugoImportResolveFunc}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + c := opts.ImportOnLoadFunc(args) + if c == "" { + return api.OnLoadResult{}, fmt.Errorf("ImportOnLoadFunc failed to resolve %q", args.Path) + } + + return api.OnLoadResult{ + ResolveDir: opts.ResolveDir, + Contents: &c, + Loader: loaderFromFilename(args.Path), + }, nil + }) + }, + } + + params := opts.Params + if params == nil { + // This way @params will always resolve to something. + params = make(map[string]any) + } + + b, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("failed to marshal params: %w", err) + } + paramsPlugin := api.Plugin{ + Name: "hugo-params-plugin", + Setup: func(build api.PluginBuild) { + build.OnResolve(api.OnResolveOptions{Filter: `^@params$`}, + func(args api.OnResolveArgs) (api.OnResolveResult, error) { + return api.OnResolveResult{ + Path: args.Importer, + Namespace: nsHugoParams, + }, nil + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsHugoParams}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + bb := b + + if opts.ImportParamsOnLoadFunc != nil { + if bbb := opts.ImportParamsOnLoadFunc(args); bbb != nil { + bb = bbb + } + } + + s := string(bb) + + return api.OnLoadResult{ + Contents: &s, + Loader: api.LoaderJSON, + }, nil + }) + }, + } + + return []api.Plugin{importResolver, paramsPlugin}, nil +} diff --git a/internal/js/esbuild/resolve_test.go b/internal/js/esbuild/resolve_test.go new file mode 100644 index 00000000000..6ea3d304b48 --- /dev/null +++ b/internal/js/esbuild/resolve_test.go @@ -0,0 +1,85 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "path" + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/testconfig" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/afero" +) + +func TestResolveComponentInAssets(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + name string + files []string + impPath string + expect string + }{ + {"Basic, extension", []string{"foo.js", "bar.js"}, "foo.js", "foo.js"}, + {"Basic, no extension", []string{"foo.js", "bar.js"}, "foo", "foo.js"}, + {"Basic, no extension, typescript", []string{"foo.ts", "bar.js"}, "foo", "foo.ts"}, + {"Not found", []string{"foo.js", "bar.js"}, "moo.js", ""}, + {"Not found, double js extension", []string{"foo.js.js", "bar.js"}, "foo.js", ""}, + {"Index file, folder only", []string{"foo/index.js", "bar.js"}, "foo", "foo/index.js"}, + {"Index file, folder and index", []string{"foo/index.js", "bar.js"}, "foo/index", "foo/index.js"}, + {"Index file, folder and index and suffix", []string{"foo/index.js", "bar.js"}, "foo/index.js", "foo/index.js"}, + {"Index ESM file, folder only", []string{"foo/index.esm.js", "bar.js"}, "foo", "foo/index.esm.js"}, + {"Index ESM file, folder and index", []string{"foo/index.esm.js", "bar.js"}, "foo/index", "foo/index.esm.js"}, + {"Index ESM file, folder and index and suffix", []string{"foo/index.esm.js", "bar.js"}, "foo/index.esm.js", "foo/index.esm.js"}, + // We added these index.esm.js cases in v0.101.0. The case below is unlikely to happen in the wild, but add a test + // to document Hugo's behavior. We pick the file with the name index.js; anything else would be breaking. + {"Index and Index ESM file, folder only", []string{"foo/index.esm.js", "foo/index.js", "bar.js"}, "foo", "foo/index.js"}, + + // Issue #8949 + {"Check file before directory", []string{"foo.js", "foo/index.js"}, "foo", "foo.js"}, + } { + c.Run(test.name, func(c *qt.C) { + baseDir := "assets" + mfs := afero.NewMemMapFs() + + for _, filename := range test.files { + c.Assert(afero.WriteFile(mfs, filepath.Join(baseDir, filename), []byte("let foo='bar';"), 0o777), qt.IsNil) + } + + conf := testconfig.GetTestConfig(mfs, config.New()) + fs := hugofs.NewFrom(mfs, conf.BaseConfig()) + + p, err := paths.New(fs, conf) + c.Assert(err, qt.IsNil) + bfs, err := filesystems.NewBase(p, nil) + c.Assert(err, qt.IsNil) + + got := resolveComponentInAssets(bfs.Assets.Fs, test.impPath) + + gotPath := "" + expect := test.expect + if got != nil { + gotPath = filepath.ToSlash(got.Filename) + expect = path.Join(baseDir, test.expect) + } + + c.Assert(gotPath, qt.Equals, expect) + }) + } +} diff --git a/lazy/init.go b/lazy/init.go index 7b88a53518b..bef3867a9a0 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -36,7 +36,7 @@ type Init struct { prev *Init children []*Init - init onceMore + init OnceMore out any err error f func(context.Context) (any, error) diff --git a/lazy/once.go b/lazy/once.go index c6abcd88493..0ebdb335aa0 100644 --- a/lazy/once.go +++ b/lazy/once.go @@ -24,13 +24,13 @@ import ( // * it can be reset, so the action can be repeated if needed // * it has methods to check if it's done or in progress -type onceMore struct { +type OnceMore struct { mu sync.Mutex lock uint32 done uint32 } -func (t *onceMore) Do(f func()) { +func (t *OnceMore) Do(f func()) { if atomic.LoadUint32(&t.done) == 1 { return } @@ -53,15 +53,15 @@ func (t *onceMore) Do(f func()) { f() } -func (t *onceMore) InProgress() bool { +func (t *OnceMore) InProgress() bool { return atomic.LoadUint32(&t.lock) == 1 } -func (t *onceMore) Done() bool { +func (t *OnceMore) Done() bool { return atomic.LoadUint32(&t.done) == 1 } -func (t *onceMore) ResetWithLock() *sync.Mutex { +func (t *OnceMore) ResetWithLock() *sync.Mutex { t.mu.Lock() defer atomic.StoreUint32(&t.done, 0) return &t.mu diff --git a/media/mediaType.go b/media/mediaType.go index a7ba1309a7d..97b10879c07 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -273,9 +273,13 @@ func (t Types) GetByType(tp string) (Type, bool) { return Type{}, false } +func (t Types) normalizeSuffix(s string) string { + return strings.ToLower(strings.TrimPrefix(s, ".")) +} + // BySuffix will return all media types matching a suffix. func (t Types) BySuffix(suffix string) []Type { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) var types []Type for _, tt := range t { if tt.hasSuffix(suffix) { @@ -287,7 +291,7 @@ func (t Types) BySuffix(suffix string) []Type { // GetFirstBySuffix will return the first type matching the given suffix. func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt, SuffixInfo{ @@ -304,7 +308,7 @@ func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { // is ambiguous. // The lookup is case insensitive. func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { if found { @@ -324,7 +328,7 @@ func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { } func (t Types) IsTextSuffix(suffix string) bool { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt.IsText() diff --git a/output/outputFormat.go b/output/outputFormat.go index d249c72b933..32fcef33c7c 100644 --- a/output/outputFormat.go +++ b/output/outputFormat.go @@ -120,6 +120,14 @@ var ( Rel: "alternate", } + GoTmplFormat = Format{ + Name: "gotmpl", + MediaType: media.Builtin.TextType, + BaseName: "index", + IsPlainText: true, + NotAlternative: true, + } + HTMLFormat = Format{ Name: "html", MediaType: media.Builtin.HTMLType, @@ -215,6 +223,7 @@ var DefaultFormats = Formats{ RobotsTxtFormat, RSSFormat, SitemapFormat, + GoTmplFormat, } func init() { diff --git a/resources/page/page.go b/resources/page/page.go index 4cda8d31fce..40bd0e43e80 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -70,8 +70,7 @@ type ChildCareProvider interface { // section. RegularPagesRecursive() Pages - // Resources returns a list of all resources. - Resources() resource.Resources + resource.ResourcesProvider } type MarkupProvider interface { diff --git a/resources/resource.go b/resources/resource.go index cc7008e5a88..b32cb0baed1 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -47,6 +47,7 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ resource.Identifier = (*genericResource)(nil) + _ resource.PathProvider = (*genericResource)(nil) _ identity.IdentityGroupProvider = (*genericResource)(nil) _ identity.DependencyManagerProvider = (*genericResource)(nil) _ identity.Identity = (*genericResource)(nil) @@ -463,6 +464,11 @@ func (l *genericResource) Key() string { return key } +// TODO1 test and document this. Consider adding it to the Resource interface. +func (l *genericResource) Path() string { + return l.paths.TargetPath() +} + func (l *genericResource) MediaType() media.Type { return l.sd.MediaType } diff --git a/resources/resource/resources.go b/resources/resource/resources.go index 32bcdbb088a..bf0b1168df0 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -16,6 +16,7 @@ package resource import ( "fmt" + "path" "strings" "github.com/gohugoio/hugo/common/paths" @@ -29,6 +30,53 @@ var _ ResourceFinder = (*Resources)(nil) // I.e. both pages and images etc. type Resources []Resource +// TODO1 +// TODO1 move to a func + template func. Maybe. +func (r Resources) Mount(base, target string) ResourceGetter { + return resourceGetterFunc(func(namev any) Resource { + name1, err := cast.ToStringE(namev) + if err != nil { + panic(err) + } + + isTargetAbs := strings.HasPrefix(target, "/") + + if target != "" { + name1 = strings.TrimPrefix(name1, target) + if !isTargetAbs { + name1 = paths.TrimLeading(name1) + } + } + + if base != "" && isTargetAbs { + name1 = path.Join(base, name1) + } + + for _, res := range r { + name2 := res.Name() + + if base != "" && !isTargetAbs { + name2 = paths.TrimLeading(strings.TrimPrefix(name2, base)) + } + + // TODO1 remove. + // fmt.Println("name1", name1, "name2", name2, "base", base) + + if strings.EqualFold(name1, name2) { + return res + } + + } + + return nil + }) +} + +type ResourcesProvider interface { + // Resources returns a list of all resources. + Resources() Resources +} + // var _ resource.ResourceFinder = (*Namespace)(nil) // ResourcesConverter converts a given slice of Resource objects to Resources. type ResourcesConverter interface { @@ -63,13 +111,25 @@ func (r Resources) Get(name any) Resource { panic(err) } - namestr = paths.AddLeadingSlash(namestr) + isDotCurrent := strings.HasPrefix(namestr, "./") + if isDotCurrent { + namestr = strings.TrimPrefix(namestr, "./") + } else { + namestr = paths.AddLeadingSlash(namestr) + } + + check := func(name string) bool { + if !isDotCurrent { + name = paths.AddLeadingSlash(name) + } + return strings.EqualFold(namestr, name) + } // First check the Name. // Note that this can be modified by the user in the front matter, // also, it does not contain any language code. for _, resource := range r { - if strings.EqualFold(namestr, paths.AddLeadingSlash(resource.Name())) { + if check(resource.Name()) { return resource } } @@ -77,7 +137,7 @@ func (r Resources) Get(name any) Resource { // Finally, check the normalized name. for _, resource := range r { if nop, ok := resource.(NameNormalizedProvider); ok { - if strings.EqualFold(namestr, paths.AddLeadingSlash(nop.NameNormalized())) { + if check(nop.NameNormalized()) { return resource } } @@ -197,14 +257,31 @@ type Source interface { Publish() error } -// ResourceFinder provides methods to find Resources. -// Note that GetRemote (as found in resources.GetRemote) is -// not covered by this interface, as this is only available as a global template function. -type ResourceFinder interface { +type ResourceGetter interface { // Get locates the Resource with the given name in the current context (e.g. in .Page.Resources). // // It returns nil if no Resource could found, panics if name is invalid. Get(name any) Resource +} + +// StaleInfoResourceGetter is a ResourceGetter that also provides information about +// whether the underlying resources are stale. +type StaleInfoResourceGetter interface { + StaleInfo + ResourceGetter +} + +type resourceGetterFunc func(name any) Resource + +func (f resourceGetterFunc) Get(name any) Resource { + return f(name) +} + +// ResourceFinder provides methods to find Resources. +// Note that GetRemote (as found in resources.GetRemote) is +// not covered by this interface, as this is only available as a global template function. +type ResourceFinder interface { + ResourceGetter // GetMatch finds the first Resource matching the given pattern, or nil if none found. // @@ -235,3 +312,69 @@ type ResourceFinder interface { // It returns nil if no Resources could found, panics if typ is invalid. ByType(typ any) Resources } + +// NewResourceGetter creates a new ResourceGetter from the given objects. +// If multiple objects are provided, they are merged into one where +// the first match wins. +func NewResourceGetter(os ...any) StaleInfoResourceGetter { + var getters multiResourceGetter + for _, o := range os { + if g, ok := unwrapResourceGetter(o); ok { + getters = append(getters, g) + } + } + + return struct { + StaleInfo + ResourceGetter + }{ + StaleInfoFunc( + func() uint32 { + for _, g := range getters { + if s, ok := g.(StaleInfo); ok { + if i := s.StaleVersion(); i > 0 { + return i + } + } + } + return 0 + }, + ), + getters, + } +} + +type multiResourceGetter []ResourceGetter + +func (m multiResourceGetter) Get(name any) Resource { + for _, g := range m { + if res := g.Get(name); res != nil { + return res + } + } + return nil +} + +func unwrapResourceGetter(v any) (ResourceGetter, bool) { + if v == nil { + return nil, false + } + switch vv := v.(type) { + case ResourceGetter: + return vv, true + case ResourcesProvider: + return vv.Resources(), true + case func(name any) Resource: + return resourceGetterFunc(vv), true + case []any: + var getters multiResourceGetter + for _, vv := range vv { + if g, ok := unwrapResourceGetter(vv); ok { + getters = append(getters, g) + } + } + return getters, len(getters) > 0 + } + + return nil, false +} diff --git a/resources/resource/resources_test.go b/resources/resource/resources_test.go new file mode 100644 index 00000000000..645bd0e4b8f --- /dev/null +++ b/resources/resource/resources_test.go @@ -0,0 +1,125 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestResourcesMount(t *testing.T) { + c := qt.New(t) + c.Assert(true, qt.IsTrue) + + var m ResourceGetter + var r Resources + + check := func(in, expect string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.Not(qt.IsNil)) + c.Assert(r.Name(), qt.Equals, expect) + } + + checkNil := func(in string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.IsNil) + } + + // Misc tests. + r = Resources{ + testResource{name: "/foo/theme.css"}, + } + + m = r.Mount("/foo", ".") + check("./theme.css", "/foo/theme.css") + + // Relative target. + r = Resources{ + testResource{name: "/a/b/c/d.txt"}, + testResource{name: "/a/b/c/e/f.txt"}, + testResource{name: "/a/b/d.txt"}, + testResource{name: "/a/b/e.txt"}, + } + + // namvev ./theme.css base /js/hugoheadlessui/components target . name /js/hugoheadlessui/components/theme.css name1 theme.css name2 . + + m = r.Mount("/a/b/c", "z") + check("z/d.txt", "/a/b/c/d.txt") + // check("./z/d.txt", "/a/b/c/d.txt") + check("z/e/f.txt", "/a/b/c/e/f.txt") + + m = r.Mount("/a/b", "") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", ".") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", "./") + check("d.txt", "/a/b/d.txt") + check("./d.txt", "/a/b/d.txt") + + m = r.Mount("/a/b", ".") + check("./d.txt", "/a/b/d.txt") + + // Absolute target. + m = r.Mount("/a/b/c", "/z") + check("/z/d.txt", "/a/b/c/d.txt") + check("/z/e/f.txt", "/a/b/c/e/f.txt") + checkNil("/z/f.txt") + + m = r.Mount("/a/b", "/z") + check("/z/c/d.txt", "/a/b/c/d.txt") + check("/z/c/e/f.txt", "/a/b/c/e/f.txt") + check("/z/d.txt", "/a/b/d.txt") + checkNil("/z/f.txt") + + m = r.Mount("", "") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + m = r.Mount("/a/b", "/a/b") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + // Resources with relative paths. + r = Resources{ + testResource{name: "a/b/c/d.txt"}, + testResource{name: "a/b/c/e/f.txt"}, + testResource{name: "a/b/d.txt"}, + testResource{name: "a/b/e.txt"}, + testResource{name: "n.txt"}, + } + + m = r.Mount("a/b", "z") + check("z/d.txt", "a/b/d.txt") + checkNil("/z/d.txt") +} + +type testResource struct { + Resource + name string +} + +func (r testResource) Name() string { + return r.name +} + +func (r testResource) NameNormalized() string { + return r.name +} diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 0fb87f37137..7684960b4be 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -108,6 +108,14 @@ type MediaTypeProvider interface { MediaType() media.Type } +// TODO1 consider removing.s +type PathProvider interface { + // Path is the relative path to this resource. + // In most cases this will be the same as the RelPermalink(), + // but it will not trigger any lazy publishing. + Path() string +} + type ResourceLinksProvider interface { // Permalink represents the absolute link to this resource. Permalink() string @@ -244,6 +252,13 @@ type StaleInfo interface { StaleVersion() uint32 } +// StaleInfoFunc is a function that returns the StaleVersion for one or more resources. +type StaleInfoFunc func() uint32 + +func (f StaleInfoFunc) StaleVersion() uint32 { + return f() +} + // StaleVersion returns the StaleVersion for the given os, // or 0 if not set. func StaleVersion(os any) uint32 { diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index cc68d225335..882ba08bda3 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,209 +14,64 @@ package js import ( - "errors" - "fmt" - "io" - "os" "path" - "path/filepath" "regexp" - "strings" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/common/text" + "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/internal/js/esbuild" - "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) // Client context for ESBuild. type Client struct { - rs *resources.Spec - sfs *filesystems.SourceFilesystem + c *esbuild.BuildClient } // New creates a new client context. func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { return &Client{ - rs: rs, - sfs: fs, + c: esbuild.NewBuildClient(fs, rs), } } -type buildTransformation struct { - optsm map[string]any - c *Client -} - -func (t *buildTransformation) Key() internal.ResourceTransformationKey { - return internal.NewResourceTransformationKey("jsbuild", t.optsm) +// Process processes a resource with the user provided options. +func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, optsm: opts}, + ) } -func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - ctx.OutMediaType = media.Builtin.JavascriptType - - opts, err := decodeOptions(t.optsm) - if err != nil { - return err - } - - if opts.TargetPath != "" { - ctx.OutPath = opts.TargetPath - } else { - ctx.ReplaceOutPathExtension(".js") +func (c *Client) transform(opts esbuild.Options, transformCtx *resources.ResourceTransformationCtx) (api.BuildResult, error) { + if transformCtx.DependencyManager != nil { + opts.DependencyManager = transformCtx.DependencyManager } - - src, err := io.ReadAll(ctx.From) - if err != nil { - return err - } - - opts.sourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) - opts.resolveDir = t.c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved - opts.contents = string(src) - opts.mediaType = ctx.InMediaType - opts.tsConfig = t.c.rs.ResolveJSConfigFile("tsconfig.json") - - buildOptions, err := toBuildOptions(opts) - if err != nil { - return err - } - - buildOptions.Plugins, err = createBuildPlugins(ctx.DependencyManager, t.c, opts) + result, err := c.c.Build(opts) if err != nil { - return err - } - - if buildOptions.Sourcemap == api.SourceMapExternal && buildOptions.Outdir == "" { - buildOptions.Outdir, err = os.MkdirTemp(os.TempDir(), "compileOutput") - if err != nil { - return err - } - defer os.Remove(buildOptions.Outdir) - } - - if opts.Inject != nil { - // Resolve the absolute filenames. - for i, ext := range opts.Inject { - impPath := filepath.FromSlash(ext) - if filepath.IsAbs(impPath) { - return fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") - } - - m := resolveComponentInAssets(t.c.rs.Assets.Fs, impPath) - - if m == nil { - return fmt.Errorf("inject: file %q not found", ext) - } - - opts.Inject[i] = m.Filename - - } - - buildOptions.Inject = opts.Inject - + return result, err } - result := api.Build(buildOptions) - - if len(result.Errors) > 0 { - - createErr := func(msg api.Message) error { - loc := msg.Location - if loc == nil { - return errors.New(msg.Text) - } - path := loc.File - if path == stdinImporter { - path = ctx.SourcePath - } - - errorMessage := msg.Text - errorMessage = strings.ReplaceAll(errorMessage, nsImportHugo+":", "") - - var ( - f afero.File - err error - ) - - if strings.HasPrefix(path, nsImportHugo) { - path = strings.TrimPrefix(path, nsImportHugo+":") - f, err = hugofs.Os.Open(path) - } else { - var fi os.FileInfo - fi, err = t.c.sfs.Fs.Stat(path) - if err == nil { - m := fi.(hugofs.FileMetaInfo).Meta() - path = m.Filename - f, err = m.Open() - } - - } - - if err == nil { - fe := herrors. - NewFileErrorFromName(errors.New(errorMessage), path). - UpdatePosition(text.Position{Offset: -1, LineNumber: loc.Line, ColumnNumber: loc.Column}). - UpdateContent(f, nil) - - f.Close() - return fe - } - - return fmt.Errorf("%s", errorMessage) - } - - var errors []error - - for _, msg := range result.Errors { - errors = append(errors, createErr(msg)) - } - - // Return 1, log the rest. - for i, err := range errors { - if i > 0 { - t.c.rs.Logger.Errorf("js.Build failed: %s", err) - } - } - - return errors[0] - } - - if buildOptions.Sourcemap == api.SourceMapExternal { + if opts.ExternalOptions.SourceMap == "external" { content := string(result.OutputFiles[1].Contents) - symPath := path.Base(ctx.OutPath) + ".map" + symPath := path.Base(transformCtx.OutPath) + ".map" re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") - if err = ctx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { - return err + if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { + return result, err } - _, err := ctx.To.Write([]byte(content)) + _, err := transformCtx.To.Write([]byte(content)) if err != nil { - return err + return result, err } } else { - _, err := ctx.To.Write(result.OutputFiles[0].Contents) + _, err := transformCtx.To.Write(result.OutputFiles[0].Contents) if err != nil { - return err + return result, err } - } - return nil -} -// Process process esbuild transform -func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { - return res.Transform( - &buildTransformation{c: c, optsm: opts}, - ) + } + return result, nil } diff --git a/resources/resource_transformers/js/build_test.go b/resources/resource_transformers/js/build_test.go deleted file mode 100644 index 30a4490edc2..00000000000 --- a/resources/resource_transformers/js/build_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package js diff --git a/resources/resource_transformers/js/js_integration_test.go b/resources/resource_transformers/js/js_integration_test.go index 304c51d3373..69c5587949c 100644 --- a/resources/resource_transformers/js/js_integration_test.go +++ b/resources/resource_transformers/js/js_integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/resources/resource_transformers/js/options.go b/resources/resource_transformers/js/options.go deleted file mode 100644 index 8c271d032d7..00000000000 --- a/resources/resource_transformers/js/options.go +++ /dev/null @@ -1,461 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package js - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/common/paths" - "github.com/gohugoio/hugo/identity" - "github.com/spf13/afero" - - "github.com/evanw/esbuild/pkg/api" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - "github.com/mitchellh/mapstructure" -) - -const ( - nsImportHugo = "ns-hugo" - nsParams = "ns-params" - - stdinImporter = "" -) - -// Options esbuild configuration -type Options struct { - // If not set, the source path will be used as the base target path. - // Note that the target path's extension may change if the target MIME type - // is different, e.g. when the source is TypeScript. - TargetPath string - - // Whether to minify to output. - Minify bool - - // Whether to write mapfiles - SourceMap string - - // The language target. - // One of: es2015, es2016, es2017, es2018, es2019, es2020 or esnext. - // Default is esnext. - Target string - - // The output format. - // One of: iife, cjs, esm - // Default is to esm. - Format string - - // External dependencies, e.g. "react". - Externals []string - - // This option allows you to automatically replace a global variable with an import from another file. - // The filenames must be relative to /assets. - // See https://esbuild.github.io/api/#inject - Inject []string - - // User defined symbols. - Defines map[string]any - - // Maps a component import to another. - Shims map[string]string - - // User defined params. Will be marshaled to JSON and available as "@params", e.g. - // import * as params from '@params'; - Params any - - // What to use instead of React.createElement. - JSXFactory string - - // What to use instead of React.Fragment. - JSXFragment string - - // What to do about JSX syntax. - // See https://esbuild.github.io/api/#jsx - JSX string - - // Which library to use to automatically import JSX helper functions from. Only works if JSX is set to automatic. - // See https://esbuild.github.io/api/#jsx-import-source - JSXImportSource string - - // There is/was a bug in WebKit with severe performance issue with the tracking - // of TDZ checks in JavaScriptCore. - // - // Enabling this flag removes the TDZ and `const` assignment checks and - // may improve performance of larger JS codebases until the WebKit fix - // is in widespread use. - // - // See https://bugs.webkit.org/show_bug.cgi?id=199866 - // Deprecated: This no longer have any effect and will be removed. - // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba - AvoidTDZ bool - - mediaType media.Type - outDir string - contents string - sourceDir string - resolveDir string - tsConfig string -} - -func decodeOptions(m map[string]any) (Options, error) { - var opts Options - - if err := mapstructure.WeakDecode(m, &opts); err != nil { - return opts, err - } - - if opts.TargetPath != "" { - opts.TargetPath = paths.ToSlashTrimLeading(opts.TargetPath) - } - - opts.Target = strings.ToLower(opts.Target) - opts.Format = strings.ToLower(opts.Format) - - return opts, nil -} - -var extensionToLoaderMap = map[string]api.Loader{ - ".js": api.LoaderJS, - ".mjs": api.LoaderJS, - ".cjs": api.LoaderJS, - ".jsx": api.LoaderJSX, - ".ts": api.LoaderTS, - ".tsx": api.LoaderTSX, - ".css": api.LoaderCSS, - ".json": api.LoaderJSON, - ".txt": api.LoaderText, -} - -func loaderFromFilename(filename string) api.Loader { - l, found := extensionToLoaderMap[filepath.Ext(filename)] - if found { - return l - } - return api.LoaderJS -} - -func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { - findFirst := func(base string) *hugofs.FileMeta { - // This is the most common sub-set of ESBuild's default extensions. - // We assume that imports of JSON, CSS etc. will be using their full - // name with extension. - for _, ext := range []string{".js", ".ts", ".tsx", ".jsx"} { - if strings.HasSuffix(impPath, ext) { - // Import of foo.js.js need the full name. - continue - } - if fi, err := fs.Stat(base + ext); err == nil { - return fi.(hugofs.FileMetaInfo).Meta() - } - } - - // Not found. - return nil - } - - var m *hugofs.FileMeta - - // We need to check if this is a regular file imported without an extension. - // There may be ambiguous situations where both foo.js and foo/index.js exists. - // This import order is in line with both how Node and ESBuild's native - // import resolver works. - - // It may be a regular file imported without an extension, e.g. - // foo or foo/index. - m = findFirst(impPath) - if m != nil { - return m - } - - base := filepath.Base(impPath) - if base == "index" { - // try index.esm.js etc. - m = findFirst(impPath + ".esm") - if m != nil { - return m - } - } - - // Check the path as is. - fi, err := fs.Stat(impPath) - - if err == nil { - if fi.IsDir() { - m = findFirst(filepath.Join(impPath, "index")) - if m == nil { - m = findFirst(filepath.Join(impPath, "index.esm")) - } - } else { - m = fi.(hugofs.FileMetaInfo).Meta() - } - } else if strings.HasSuffix(base, ".js") { - m = findFirst(strings.TrimSuffix(impPath, ".js")) - } - - return m -} - -func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ([]api.Plugin, error) { - fs := c.rs.Assets - - resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { - impPath := args.Path - if opts.Shims != nil { - override, found := opts.Shims[impPath] - if found { - impPath = override - } - } - isStdin := args.Importer == stdinImporter - var relDir string - if !isStdin { - rel, found := fs.MakePathRelative(args.Importer, true) - if !found { - // Not in any of the /assets folders. - // This is an import from a node_modules, let - // ESBuild resolve this. - return api.OnResolveResult{}, nil - } - - relDir = filepath.Dir(rel) - } else { - relDir = opts.sourceDir - } - - // Imports not starting with a "." is assumed to live relative to /assets. - // Hugo makes no assumptions about the directory structure below /assets. - if relDir != "" && strings.HasPrefix(impPath, ".") { - impPath = filepath.Join(relDir, impPath) - } - - m := resolveComponentInAssets(fs.Fs, impPath) - - if m != nil { - depsManager.AddIdentity(m.PathInfo) - - // Store the source root so we can create a jsconfig.json - // to help IntelliSense when the build is done. - // This should be a small number of elements, and when - // in server mode, we may get stale entries on renames etc., - // but that shouldn't matter too much. - c.rs.JSConfigBuilder.AddSourceRoot(m.SourceRoot) - return api.OnResolveResult{Path: m.Filename, Namespace: nsImportHugo}, nil - } - - // Fall back to ESBuild's resolve. - return api.OnResolveResult{}, nil - } - - importResolver := api.Plugin{ - Name: "hugo-import-resolver", - Setup: func(build api.PluginBuild) { - build.OnResolve(api.OnResolveOptions{Filter: `.*`}, - func(args api.OnResolveArgs) (api.OnResolveResult, error) { - return resolveImport(args) - }) - build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsImportHugo}, - func(args api.OnLoadArgs) (api.OnLoadResult, error) { - b, err := os.ReadFile(args.Path) - if err != nil { - return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) - } - c := string(b) - return api.OnLoadResult{ - // See https://github.com/evanw/esbuild/issues/502 - // This allows all modules to resolve dependencies - // in the main project's node_modules. - ResolveDir: opts.resolveDir, - Contents: &c, - Loader: loaderFromFilename(args.Path), - }, nil - }) - }, - } - - params := opts.Params - if params == nil { - // This way @params will always resolve to something. - params = make(map[string]any) - } - - b, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("failed to marshal params: %w", err) - } - bs := string(b) - paramsPlugin := api.Plugin{ - Name: "hugo-params-plugin", - Setup: func(build api.PluginBuild) { - build.OnResolve(api.OnResolveOptions{Filter: `^@params$`}, - func(args api.OnResolveArgs) (api.OnResolveResult, error) { - return api.OnResolveResult{ - Path: args.Path, - Namespace: nsParams, - }, nil - }) - build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsParams}, - func(args api.OnLoadArgs) (api.OnLoadResult, error) { - return api.OnLoadResult{ - Contents: &bs, - Loader: api.LoaderJSON, - }, nil - }) - }, - } - - return []api.Plugin{importResolver, paramsPlugin}, nil -} - -func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { - var target api.Target - switch opts.Target { - case "", "esnext": - target = api.ESNext - case "es5": - target = api.ES5 - case "es6", "es2015": - target = api.ES2015 - case "es2016": - target = api.ES2016 - case "es2017": - target = api.ES2017 - case "es2018": - target = api.ES2018 - case "es2019": - target = api.ES2019 - case "es2020": - target = api.ES2020 - case "es2021": - target = api.ES2021 - case "es2022": - target = api.ES2022 - case "es2023": - target = api.ES2023 - default: - err = fmt.Errorf("invalid target: %q", opts.Target) - return - } - - mediaType := opts.mediaType - if mediaType.IsZero() { - mediaType = media.Builtin.JavascriptType - } - - var loader api.Loader - switch mediaType.SubType { - // TODO(bep) ESBuild support a set of other loaders, but I currently fail - // to see the relevance. That may change as we start using this. - case media.Builtin.JavascriptType.SubType: - loader = api.LoaderJS - case media.Builtin.TypeScriptType.SubType: - loader = api.LoaderTS - case media.Builtin.TSXType.SubType: - loader = api.LoaderTSX - case media.Builtin.JSXType.SubType: - loader = api.LoaderJSX - default: - err = fmt.Errorf("unsupported Media Type: %q", opts.mediaType) - return - } - - var format api.Format - // One of: iife, cjs, esm - switch opts.Format { - case "", "iife": - format = api.FormatIIFE - case "esm": - format = api.FormatESModule - case "cjs": - format = api.FormatCommonJS - default: - err = fmt.Errorf("unsupported script output format: %q", opts.Format) - return - } - - var jsx api.JSX - switch opts.JSX { - case "", "transform": - jsx = api.JSXTransform - case "preserve": - jsx = api.JSXPreserve - case "automatic": - jsx = api.JSXAutomatic - default: - err = fmt.Errorf("unsupported jsx type: %q", opts.JSX) - return - } - - var defines map[string]string - if opts.Defines != nil { - defines = maps.ToStringMapString(opts.Defines) - } - - // By default we only need to specify outDir and no outFile - outDir := opts.outDir - outFile := "" - var sourceMap api.SourceMap - switch opts.SourceMap { - case "inline": - sourceMap = api.SourceMapInline - case "external": - sourceMap = api.SourceMapExternal - case "": - sourceMap = api.SourceMapNone - default: - err = fmt.Errorf("unsupported sourcemap type: %q", opts.SourceMap) - return - } - - buildOptions = api.BuildOptions{ - Outfile: outFile, - Bundle: true, - - Target: target, - Format: format, - Sourcemap: sourceMap, - - MinifyWhitespace: opts.Minify, - MinifyIdentifiers: opts.Minify, - MinifySyntax: opts.Minify, - - Outdir: outDir, - Define: defines, - - External: opts.Externals, - - JSXFactory: opts.JSXFactory, - JSXFragment: opts.JSXFragment, - - JSX: jsx, - JSXImportSource: opts.JSXImportSource, - - Tsconfig: opts.tsConfig, - - // Note: We're not passing Sourcefile to ESBuild. - // This makes ESBuild pass `stdin` as the Importer to the import - // resolver, which is what we need/expect. - Stdin: &api.StdinOptions{ - Contents: opts.contents, - ResolveDir: opts.resolveDir, - Loader: loader, - }, - } - return -} diff --git a/resources/resource_transformers/js/options_test.go b/resources/resource_transformers/js/options_test.go deleted file mode 100644 index 53aa9b6bbff..00000000000 --- a/resources/resource_transformers/js/options_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package js - -import ( - "path" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/config/testconfig" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/hugolib/paths" - "github.com/gohugoio/hugo/media" - - "github.com/spf13/afero" - - "github.com/evanw/esbuild/pkg/api" - - qt "github.com/frankban/quicktest" -) - -// This test is added to test/warn against breaking the "stability" of the -// cache key. It's sometimes needed to break this, but should be avoided if possible. -func TestOptionKey(t *testing.T) { - c := qt.New(t) - - opts := map[string]any{ - "TargetPath": "foo", - "Target": "es2018", - } - - key := (&buildTransformation{optsm: opts}).Key() - - c.Assert(key.Value(), qt.Equals, "jsbuild_1533819657654811600") -} - -func TestToBuildOptions(t *testing.T) { - c := qt.New(t) - - opts, err := toBuildOptions(Options{mediaType: media.Builtin.JavascriptType}) - - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ESNext, - Format: api.FormatIIFE, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", - Format: "cjs", - Minify: true, - mediaType: media.Builtin.JavascriptType, - AvoidTDZ: true, - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapInline, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapInline, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "external", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapExternal, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - mediaType: media.Builtin.JavascriptType, - JSX: "automatic", JSXImportSource: "preact", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ESNext, - Format: api.FormatIIFE, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - JSX: api.JSXAutomatic, - JSXImportSource: "preact", - }) -} - -func TestToBuildOptionsTarget(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - target string - expect api.Target - }{ - {"es2015", api.ES2015}, - {"es2016", api.ES2016}, - {"es2017", api.ES2017}, - {"es2018", api.ES2018}, - {"es2019", api.ES2019}, - {"es2020", api.ES2020}, - {"es2021", api.ES2021}, - {"es2022", api.ES2022}, - {"es2023", api.ES2023}, - {"", api.ESNext}, - {"esnext", api.ESNext}, - } { - c.Run(test.target, func(c *qt.C) { - opts, err := toBuildOptions(Options{ - Target: test.target, - mediaType: media.Builtin.JavascriptType, - }) - c.Assert(err, qt.IsNil) - c.Assert(opts.Target, qt.Equals, test.expect) - }) - } -} - -func TestResolveComponentInAssets(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - name string - files []string - impPath string - expect string - }{ - {"Basic, extension", []string{"foo.js", "bar.js"}, "foo.js", "foo.js"}, - {"Basic, no extension", []string{"foo.js", "bar.js"}, "foo", "foo.js"}, - {"Basic, no extension, typescript", []string{"foo.ts", "bar.js"}, "foo", "foo.ts"}, - {"Not found", []string{"foo.js", "bar.js"}, "moo.js", ""}, - {"Not found, double js extension", []string{"foo.js.js", "bar.js"}, "foo.js", ""}, - {"Index file, folder only", []string{"foo/index.js", "bar.js"}, "foo", "foo/index.js"}, - {"Index file, folder and index", []string{"foo/index.js", "bar.js"}, "foo/index", "foo/index.js"}, - {"Index file, folder and index and suffix", []string{"foo/index.js", "bar.js"}, "foo/index.js", "foo/index.js"}, - {"Index ESM file, folder only", []string{"foo/index.esm.js", "bar.js"}, "foo", "foo/index.esm.js"}, - {"Index ESM file, folder and index", []string{"foo/index.esm.js", "bar.js"}, "foo/index", "foo/index.esm.js"}, - {"Index ESM file, folder and index and suffix", []string{"foo/index.esm.js", "bar.js"}, "foo/index.esm.js", "foo/index.esm.js"}, - // We added these index.esm.js cases in v0.101.0. The case below is unlikely to happen in the wild, but add a test - // to document Hugo's behavior. We pick the file with the name index.js; anything else would be breaking. - {"Index and Index ESM file, folder only", []string{"foo/index.esm.js", "foo/index.js", "bar.js"}, "foo", "foo/index.js"}, - - // Issue #8949 - {"Check file before directory", []string{"foo.js", "foo/index.js"}, "foo", "foo.js"}, - } { - c.Run(test.name, func(c *qt.C) { - baseDir := "assets" - mfs := afero.NewMemMapFs() - - for _, filename := range test.files { - c.Assert(afero.WriteFile(mfs, filepath.Join(baseDir, filename), []byte("let foo='bar';"), 0o777), qt.IsNil) - } - - conf := testconfig.GetTestConfig(mfs, config.New()) - fs := hugofs.NewFrom(mfs, conf.BaseConfig()) - - p, err := paths.New(fs, conf) - c.Assert(err, qt.IsNil) - bfs, err := filesystems.NewBase(p, nil) - c.Assert(err, qt.IsNil) - - got := resolveComponentInAssets(bfs.Assets.Fs, test.impPath) - - gotPath := "" - expect := test.expect - if got != nil { - gotPath = filepath.ToSlash(got.Filename) - expect = path.Join(baseDir, test.expect) - } - - c.Assert(gotPath, qt.Equals, expect) - }) - } -} diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go new file mode 100644 index 00000000000..13909e54cc6 --- /dev/null +++ b/resources/resource_transformers/js/transform.go @@ -0,0 +1,68 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "io" + "path" + "path/filepath" + + "github.com/gohugoio/hugo/internal/js/esbuild" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" +) + +type buildTransformation struct { + optsm map[string]any + c *Client +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("jsbuild", t.optsm) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + ctx.OutMediaType = media.Builtin.JavascriptType + + var opts esbuild.Options + + if t.optsm != nil { + optsExt, err := esbuild.DecodeExternalOptions(t.optsm) + if err != nil { + return err + } + opts.ExternalOptions = optsExt + } + + if opts.TargetPath != "" { + ctx.OutPath = opts.TargetPath + } else { + ctx.ReplaceOutPathExtension(".js") + } + + src, err := io.ReadAll(ctx.From) + if err != nil { + return err + } + + opts.SourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) + opts.Contents = string(src) + opts.MediaType = ctx.InMediaType + opts.Stdin = true + + _, err = t.c.transform(opts, ctx) + + return err +} diff --git a/resources/transform.go b/resources/transform.go index 336495e6d07..9781ea6c31c 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -52,6 +52,7 @@ var ( _ identity.IdentityGroupProvider = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) + _ resource.PathProvider = (*resourceAdapter)(nil) _ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil) _ resource.WithResourceMetaProvider = (*resourceAdapter)(nil) _ identity.DependencyManagerProvider = (*resourceAdapter)(nil) @@ -277,6 +278,11 @@ func (r *resourceAdapter) Key() string { return r.target.(resource.Identifier).Key() } +func (r *resourceAdapter) Path() string { + r.init(false, false) + return r.target.(resource.PathProvider).Path() +} + func (r *resourceAdapter) MediaType() media.Type { r.init(false, false) return r.target.MediaType() diff --git a/tpl/js/init.go b/tpl/js/init.go index 69d7c7275d6..97e76c33d87 100644 --- a/tpl/js/init.go +++ b/tpl/js/init.go @@ -24,7 +24,10 @@ const name = "js" func init() { f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { - ctx := New(d) + ctx, err := New(d) + if err != nil { + panic(err) + } ns := &internal.TemplateFuncsNamespace{ Name: name, diff --git a/tpl/js/js.go b/tpl/js/js.go index c68e0af9272..02f014c739d 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,30 +16,48 @@ package js import ( "errors" + "path" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/babel" "github.com/gohugoio/hugo/resources/resource_transformers/js" + jstransform "github.com/gohugoio/hugo/resources/resource_transformers/js" "github.com/gohugoio/hugo/tpl/internal/resourcehelpers" ) // New returns a new instance of the js-namespaced template functions. -func New(deps *deps.Deps) *Namespace { +func New(deps *deps.Deps) (*Namespace, error) { if deps.ResourceSpec == nil { - return &Namespace{} + return &Namespace{}, nil } - return &Namespace{ - client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), - babelClient: babel.New(deps.ResourceSpec), + + batcherClient, err := esbuild.NewBatcherClient(deps) + if err != nil { + return nil, err } + + return &Namespace{ + d: deps, + jsTransformClient: jstransform.New(deps.BaseFs.Assets, deps.ResourceSpec), + jsBatcherClient: batcherClient, + createClient: create.New(deps.ResourceSpec), + babelClient: babel.New(deps.ResourceSpec), + }, nil } // Namespace provides template functions for the "js" namespace. type Namespace struct { - client *js.Client - babelClient *babel.Client + d *deps.Deps + + jsTransformClient *js.Client + createClient *create.Client + babelClient *babel.Client + jsBatcherClient *esbuild.BatcherClient } // Build processes the given Resource with ESBuild. @@ -65,7 +83,18 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { m = map[string]any{"targetPath": targetPath} } - return ns.client.Process(r, m) + return ns.jsTransformClient.Process(r, m) +} + +func (ns *Namespace) Batch(id string, store *maps.Scratch) (esbuild.Batcher, error) { + key := path.Join(esbuild.NsBatch, id) + b, err := store.GetOrCreate(key, func() (any, error) { + return ns.jsBatcherClient.New(id) + }) + if err != nil { + return nil, err + } + return b.(esbuild.Batcher), nil } // Babel processes the given Resource with Babel. diff --git a/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl b/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl new file mode 100644 index 00000000000..e1b3b9bc3a9 --- /dev/null +++ b/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl @@ -0,0 +1,16 @@ +{{ range $i, $e := .Scripts -}} + {{ printf "import { %s as Script%d } from %q;" .Export $i .Import }} +{{ end -}} +{{ range $i, $e := .Runners }} + {{ printf "import { %s as Run%d } from %q;" .Export $i .Import }} +{{ end }} +{{/* */}} +let scripts = []; +{{ range $i, $e := .Scripts -}} + scripts.push({{ .RunnerJSON $i }}); +{{ end -}} +{{/* */}} +{{ range $i, $e := .Runners }} + {{ $id := printf "Run%d" $i }} + {{ $id }}(scripts); +{{ end }}