Go 1.21 Upgrade

Fixes #11351

tpl/internal/go_templates/testenv/exec.go

@@ -6,33 +6,78 @@ package testenv
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
-// HasExec reports whether the current system can start new processes
-// using os.StartProcess or (more commonly) exec.Command.
-func HasExec() bool {
-	switch runtime.GOOS {
-	case "js", "ios":
-		return false
-	}
-	return true
-}
-
 // MustHaveExec checks that the current system can start new processes
 // using os.StartProcess or (more commonly) exec.Command.
 // If not, MustHaveExec calls t.Skip with an explanation.
+//
+// On some platforms MustHaveExec checks for exec support by re-executing the
+// current executable, which must be a binary built by 'go test'.
+// We intentionally do not provide a HasExec function because of the risk of
+// inappropriate recursion in TestMain functions.
+//
+// To check for exec support outside of a test, just try to exec the command.
+// If exec is not supported, testenv.SyscallIsNotSupported will return true
+// for the resulting error.
 func MustHaveExec(t testing.TB) {
-	if !HasExec() {
-		t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH)
+	tryExecOnce.Do(func() {
+		tryExecErr = tryExec()
+	})
+	if tryExecErr != nil {
+		t.Skipf("skipping test: cannot exec subprocess on %s/%s: %v", runtime.GOOS, runtime.GOARCH, tryExecErr)
 	}
 }
 
+var (
+	tryExecOnce sync.Once
+	tryExecErr  error
+)
+
+func tryExec() error {
+	switch runtime.GOOS {
+	case "wasip1", "js", "ios":
+	default:
+		// Assume that exec always works on non-mobile platforms and Android.
+		return nil
+	}
+
+	// ios has an exec syscall but on real iOS devices it might return a
+	// permission error. In an emulated environment (such as a Corellium host)
+	// it might succeed, so if we need to exec we'll just have to try it and
+	// find out.
+	//
+	// As of 2023-04-19 wasip1 and js don't have exec syscalls at all, but we
+	// may as well use the same path so that this branch can be tested without
+	// an ios environment.
+
+	/*if !testing.Testing() {
+		// This isn't a standard 'go test' binary, so we don't know how to
+		// self-exec in a way that should succeed without side effects.
+		// Just forget it.
+		return errors.New("can't probe for exec support with a non-test executable")
+	}*/
+
+	// We know that this is a test executable. We should be able to run it with a
+	// no-op flag to check for overall exec support.
+	exe, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("can't probe for exec support: %w", err)
+	}
+	cmd := exec.Command(exe, "-test.list=^$")
+	cmd.Env = origEnv
+	return cmd.Run()
+}
+
 var execPaths sync.Map // path -> error
 
 // MustHaveExecPath checks that the current system can start the named executable
@@ -82,7 +127,87 @@ func CleanCmdEnv(cmd *exec.Cmd) *exec.Cmd {
 //   - fails the test if the command does not complete before the test's deadline, and
 //   - sets a Cleanup function that verifies that the test did not leak a subprocess.
 func CommandContext(t testing.TB, ctx context.Context, name string, args ...string) *exec.Cmd {
-	panic("Not implemented, Hugo is not using this")
+	t.Helper()
+	MustHaveExec(t)
+
+	var (
+		cancelCtx   context.CancelFunc
+		gracePeriod time.Duration // unlimited unless the test has a deadline (to allow for interactive debugging)
+	)
+
+	if t, ok := t.(interface {
+		testing.TB
+		Deadline() (time.Time, bool)
+	}); ok {
+		if td, ok := t.Deadline(); ok {
+			// Start with a minimum grace period, just long enough to consume the
+			// output of a reasonable program after it terminates.
+			gracePeriod = 100 * time.Millisecond
+			if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" {
+				scale, err := strconv.Atoi(s)
+				if err != nil {
+					t.Fatalf("invalid GO_TEST_TIMEOUT_SCALE: %v", err)
+				}
+				gracePeriod *= time.Duration(scale)
+			}
+
+			// If time allows, increase the termination grace period to 5% of the
+			// test's remaining time.
+			testTimeout := time.Until(td)
+			if gp := testTimeout / 20; gp > gracePeriod {
+				gracePeriod = gp
+			}
+
+			// When we run commands that execute subprocesses, we want to reserve two
+			// grace periods to clean up: one for the delay between the first
+			// termination signal being sent (via the Cancel callback when the Context
+			// expires) and the process being forcibly terminated (via the WaitDelay
+			// field), and a second one for the delay between the process being
+			// terminated and the test logging its output for debugging.
+			//
+			// (We want to ensure that the test process itself has enough time to
+			// log the output before it is also terminated.)
+			cmdTimeout := testTimeout - 2*gracePeriod
+
+			if cd, ok := ctx.Deadline(); !ok || time.Until(cd) > cmdTimeout {
+				// Either ctx doesn't have a deadline, or its deadline would expire
+				// after (or too close before) the test has already timed out.
+				// Add a shorter timeout so that the test will produce useful output.
+				ctx, cancelCtx = context.WithTimeout(ctx, cmdTimeout)
+			}
+		}
+	}
+
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Cancel = func() error {
+		if cancelCtx != nil && ctx.Err() == context.DeadlineExceeded {
+			// The command timed out due to running too close to the test's deadline.
+			// There is no way the test did that intentionally — it's too close to the
+			// wire! — so mark it as a test failure. That way, if the test expects the
+			// command to fail for some other reason, it doesn't have to distinguish
+			// between that reason and a timeout.
+			t.Errorf("test timed out while running command: %v", cmd)
+		} else {
+			// The command is being terminated due to ctx being canceled, but
+			// apparently not due to an explicit test deadline that we added.
+			// Log that information in case it is useful for diagnosing a failure,
+			// but don't actually fail the test because of it.
+			t.Logf("%v: terminating command: %v", ctx.Err(), cmd)
+		}
+		return cmd.Process.Signal(Sigquit)
+	}
+	cmd.WaitDelay = gracePeriod
+
+	t.Cleanup(func() {
+		if cancelCtx != nil {
+			cancelCtx()
+		}
+		if cmd.Process != nil && cmd.ProcessState == nil {
+			t.Errorf("command was started, but test did not wait for it to complete: %v", cmd)
+		}
+	})
+
+	return cmd
 }
 
 // Command is like exec.Command, but applies the same changes as

tpl/internal/go_templates/testenv/testenv.go

@@ -11,9 +11,14 @@
 package testenv
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
+
+	"github.com/gohugoio/hugo/tpl/internal/go_templates/cfg"
+
+	//"internal/platform"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -22,10 +27,14 @@ import (
 	"strings"
 	"sync"
 	"testing"
-
-	"github.com/gohugoio/hugo/tpl/internal/go_templates/cfg"
 )
 
+// Save the original environment during init for use in checks. A test
+// binary may modify its environment before calling HasExec to change its
+// behavior (such as mimicking a command-line tool), and that modified
+// environment might cause environment checks to behave erratically.
+var origEnv = os.Environ()
+
 // Builder reports the name of the builder running this test
 // (for example, "linux-amd64" or "windows-386-gce").
 // If the test is not running on the build infrastructure,
@@ -37,29 +46,26 @@ func Builder() string {
 // HasGoBuild reports whether the current system can build programs with “go build”
 // and then run them with os.StartProcess or exec.Command.
 func HasGoBuild() bool {
-	if os.Getenv("GO_GCFLAGS") != "" {
-		// It's too much work to require every caller of the go command
-		// to pass along "-gcflags="+os.Getenv("GO_GCFLAGS").
-		// For now, if $GO_GCFLAGS is set, report that we simply can't
-		// run go build.
-		return false
-	}
-	switch runtime.GOOS {
-	case "android", "js", "ios":
-		return false
-	}
-	return true
+	// Modified by Hugo (not needed)
+	return false
 }
 
+var (
+	goBuildOnce sync.Once
+	goBuildErr  error
+)
+
 // MustHaveGoBuild checks that the current system can build programs with “go build”
 // and then run them with os.StartProcess or exec.Command.
 // If not, MustHaveGoBuild calls t.Skip with an explanation.
 func MustHaveGoBuild(t testing.TB) {
 	if os.Getenv("GO_GCFLAGS") != "" {
+		t.Helper()
 		t.Skipf("skipping test: 'go build' not compatible with setting $GO_GCFLAGS")
 	}
 	if !HasGoBuild() {
-		t.Skipf("skipping test: 'go build' not available on %s/%s", runtime.GOOS, runtime.GOARCH)
+		t.Helper()
+		t.Skipf("skipping test: 'go build' unavailable: %v", goBuildErr)
 	}
 }
 
@@ -77,6 +83,25 @@ func MustHaveGoRun(t testing.TB) {
 	}
 }
 
+// HasParallelism reports whether the current system can execute multiple
+// threads in parallel.
+// There is a copy of this function in cmd/dist/test.go.
+func HasParallelism() bool {
+	switch runtime.GOOS {
+	case "js", "wasip1":
+		return false
+	}
+	return true
+}
+
+// MustHaveParallelism checks that the current system can execute multiple
+// threads in parallel. If not, MustHaveParallelism calls t.Skip with an explanation.
+func MustHaveParallelism(t testing.TB) {
+	if !HasParallelism() {
+		t.Skipf("skipping test: no parallelism available on %s/%s", runtime.GOOS, runtime.GOARCH)
+	}
+}
+
 // GoToolPath reports the path to the Go tool.
 // It is a convenience wrapper around GoTool.
 // If the tool is unavailable GoToolPath calls t.Skip.
@@ -124,6 +149,9 @@ func findGOROOT() (string, error) {
 		// runs the test in the directory containing the packaged under test.) That
 		// means that if we start walking up the tree, we should eventually find
 		// GOROOT/src/go.mod, and we can report the parent directory of that.
+		//
+		// Notably, this works even if we can't run 'go env GOROOT' as a
+		// subprocess.
 
 		cwd, err := os.Getwd()
 		if err != nil {
@@ -174,7 +202,8 @@ func findGOROOT() (string, error) {
 
 // GOROOT reports the path to the directory containing the root of the Go
 // project source tree. This is normally equivalent to runtime.GOROOT, but
-// works even if the test binary was built with -trimpath.
+// works even if the test binary was built with -trimpath and cannot exec
+// 'go env GOROOT'.
 //
 // If GOROOT cannot be found, GOROOT skips t if t is non-nil,
 // or panics otherwise.
@@ -195,25 +224,18 @@ func GoTool() (string, error) {
 	if !HasGoBuild() {
 		return "", errors.New("platform cannot run go tool")
 	}
-	var exeSuffix string
-	if runtime.GOOS == "windows" {
-		exeSuffix = ".exe"
-	}
-	goroot, err := findGOROOT()
-	if err != nil {
-		return "", fmt.Errorf("cannot find go tool: %w", err)
-	}
-	path := filepath.Join(goroot, "bin", "go"+exeSuffix)
-	if _, err := os.Stat(path); err == nil {
-		return path, nil
-	}
-	goBin, err := exec.LookPath("go" + exeSuffix)
-	if err != nil {
-		return "", errors.New("cannot find go tool: " + err.Error())
-	}
-	return goBin, nil
+	goToolOnce.Do(func() {
+		goToolPath, goToolErr = exec.LookPath("go")
+	})
+	return goToolPath, goToolErr
 }
 
+var (
+	goToolOnce sync.Once
+	goToolPath string
+	goToolErr  error
+)
+
 // HasSrc reports whether the entire source tree is available under GOROOT.
 func HasSrc() bool {
 	switch runtime.GOOS {
@@ -226,50 +248,82 @@ func HasSrc() bool {
 // HasExternalNetwork reports whether the current system can use
 // external (non-localhost) networks.
 func HasExternalNetwork() bool {
-	return !testing.Short() && runtime.GOOS != "js"
+	return !testing.Short() && runtime.GOOS != "js" && runtime.GOOS != "wasip1"
 }
 
 // MustHaveExternalNetwork checks that the current system can use
 // external (non-localhost) networks.
 // If not, MustHaveExternalNetwork calls t.Skip with an explanation.
 func MustHaveExternalNetwork(t testing.TB) {
-	if runtime.GOOS == "js" {
+	if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
+		t.Helper()
 		t.Skipf("skipping test: no external network on %s", runtime.GOOS)
 	}
 	if testing.Short() {
+		t.Helper()
 		t.Skipf("skipping test: no external network in -short mode")
 	}
 }
 
-var haveCGO bool
-
 // HasCGO reports whether the current system can use cgo.
 func HasCGO() bool {
-	return haveCGO
+	hasCgoOnce.Do(func() {
+		goTool, err := GoTool()
+		if err != nil {
+			return
+		}
+		cmd := exec.Command(goTool, "env", "CGO_ENABLED")
+		cmd.Env = origEnv
+		out, err := cmd.Output()
+		if err != nil {
+			panic(fmt.Sprintf("%v: %v", cmd, out))
+		}
+		hasCgo, err = strconv.ParseBool(string(bytes.TrimSpace(out)))
+		if err != nil {
+			panic(fmt.Sprintf("%v: non-boolean output %q", cmd, out))
+		}
+	})
+	return hasCgo
 }
 
+var (
+	hasCgoOnce sync.Once
+	hasCgo     bool
+)
+
 // MustHaveCGO calls t.Skip if cgo is not available.
 func MustHaveCGO(t testing.TB) {
-	if !haveCGO {
+	if !HasCGO() {
 		t.Skipf("skipping test: no cgo")
 	}
 }
 
 // CanInternalLink reports whether the current system can link programs with
 // internal linking.
-func CanInternalLink() bool {
-	panic("not implemented, not needed by Hugo")
+func CanInternalLink(withCgo bool) bool {
+	// Modified by Hugo (not needed)
+	return false
 }
 
 // MustInternalLink checks that the current system can link programs with internal
 // linking.
 // If not, MustInternalLink calls t.Skip with an explanation.
-func MustInternalLink(t testing.TB) {
-	if !CanInternalLink() {
+func MustInternalLink(t testing.TB, withCgo bool) {
+	if !CanInternalLink(withCgo) {
+		if withCgo && CanInternalLink(false) {
+			t.Skipf("skipping test: internal linking on %s/%s is not supported with cgo", runtime.GOOS, runtime.GOARCH)
+		}
 		t.Skipf("skipping test: internal linking on %s/%s is not supported", runtime.GOOS, runtime.GOARCH)
 	}
 }
 
+// MustHaveBuildMode reports whether the current system can build programs in
+// the given build mode.
+// If not, MustHaveBuildMode calls t.Skip with an explanation.
+func MustHaveBuildMode(t testing.TB, buildmode string) {
+	// Modified by Hugo (not needed)
+}
+
 // HasSymlink reports whether the current system can use os.Symlink.
 func HasSymlink() bool {
 	ok, _ := hasSymlink()
@@ -281,7 +335,7 @@ func HasSymlink() bool {
 func MustHaveSymlink(t testing.TB) {
 	ok, reason := hasSymlink()
 	if !ok {
-		t.Skipf("skipping test: cannot make symlinks on %s/%s%s", runtime.GOOS, runtime.GOARCH, reason)
+		t.Skipf("skipping test: cannot make symlinks on %s/%s: %s", runtime.GOOS, runtime.GOARCH, reason)
 	}
 }
 
@@ -320,7 +374,7 @@ func SkipFlakyNet(t testing.TB) {
 // CPUIsSlow reports whether the CPU running the test is suspected to be slow.
 func CPUIsSlow() bool {
 	switch runtime.GOARCH {
-	case "arm", "mips", "mipsle", "mips64", "mips64le":
+	case "arm", "mips", "mipsle", "mips64", "mips64le", "wasm":
 		return true
 	}
 	return false
@@ -346,8 +400,51 @@ func SkipIfOptimizationOff(t testing.TB) {
 }
 
 // WriteImportcfg writes an importcfg file used by the compiler or linker to
-// dstPath containing entries for the packages in std and cmd in addition
-// to the package to package file mappings in additionalPackageFiles.
-func WriteImportcfg(t testing.TB, dstPath string, additionalPackageFiles map[string]string) {
-	panic("not implemented, not needed by Hugo")
+// dstPath containing entries for the file mappings in packageFiles, as well
+// as for the packages transitively imported by the package(s) in pkgs.
+//
+// pkgs may include any package pattern that is valid to pass to 'go list',
+// so it may also be a list of Go source files all in the same directory.
+func WriteImportcfg(t testing.TB, dstPath string, packageFiles map[string]string, pkgs ...string) {
+	t.Helper()
+
+	icfg := new(bytes.Buffer)
+	icfg.WriteString("# import config\n")
+	for k, v := range packageFiles {
+		fmt.Fprintf(icfg, "packagefile %s=%s\n", k, v)
+	}
+
+	if len(pkgs) > 0 {
+		// Use 'go list' to resolve any missing packages and rewrite the import map.
+		cmd := Command(t, GoToolPath(t), "list", "-export", "-deps", "-f", `{{if ne .ImportPath "command-line-arguments"}}{{if .Export}}{{.ImportPath}}={{.Export}}{{end}}{{end}}`)
+		cmd.Args = append(cmd.Args, pkgs...)
+		cmd.Stderr = new(strings.Builder)
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatalf("%v: %v\n%s", cmd, err, cmd.Stderr)
+		}
+
+		for _, line := range strings.Split(string(out), "\n") {
+			if line == "" {
+				continue
+			}
+			importPath, export, ok := strings.Cut(line, "=")
+			if !ok {
+				t.Fatalf("invalid line in output from %v:\n%s", cmd, line)
+			}
+			if packageFiles[importPath] == "" {
+				fmt.Fprintf(icfg, "packagefile %s=%s\n", importPath, export)
+			}
+		}
+	}
+
+	if err := os.WriteFile(dstPath, icfg.Bytes(), 0666); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// SyscallIsNotSupported reports whether err may indicate that a system call is
+// not supported by the current platform or execution environment.
+func SyscallIsNotSupported(err error) bool {
+	return syscallIsNotSupported(err)
 }

tpl/internal/go_templates/testenv/testenv_cgo.go

@@ -1,11 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build cgo
-
-package testenv
-
-func init() {
-	haveCGO = true
-}

tpl/internal/go_templates/testenv/testenv_notunix.go

@@ -2,12 +2,19 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build windows || plan9 || (js && wasm)
+//go:build windows || plan9 || (js && wasm) || wasip1
 
 package testenv
 
-import "os"
+import (
+	"os"
+)
 
 // Sigquit is the signal to send to kill a hanging subprocess.
 // On Unix we send SIGQUIT, but on non-Unix we only have os.Kill.
 var Sigquit = os.Kill
+
+func syscallIsNotSupported(err error) bool {
+	// Removed by Hugo (not supported in Go 1.20).
+	return false
+}

tpl/internal/go_templates/testenv/testenv_notwin.go

@@ -7,13 +7,39 @@
 package testenv
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"runtime"
 )
 
 func hasSymlink() (ok bool, reason string) {
 	switch runtime.GOOS {
-	case "android", "plan9":
+	case "plan9":
 		return false, ""
+	case "android", "wasip1":
+		// For wasip1, some runtimes forbid absolute symlinks,
+		// or symlinks that escape the current working directory.
+		// Perform a simple test to see whether the runtime
+		// supports symlinks or not. If we get a permission
+		// error, the runtime does not support symlinks.
+		dir, err := os.MkdirTemp("", "")
+		if err != nil {
+			return false, ""
+		}
+		defer func() {
+			_ = os.RemoveAll(dir)
+		}()
+		fpath := filepath.Join(dir, "testfile.txt")
+		if err := os.WriteFile(fpath, nil, 0644); err != nil {
+			return false, ""
+		}
+		if err := os.Symlink(fpath, filepath.Join(dir, "testlink")); err != nil {
+			if SyscallIsNotSupported(err) {
+				return false, fmt.Sprintf("symlinks unsupported: %s", err.Error())
+			}
+			return false, ""
+		}
 	}
 
 	return true, ""

tpl/internal/go_templates/testenv/testenv_test.go

@@ -5,16 +5,16 @@
 package testenv_test
 
 import (
+	"github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"
+	//"internal/platform"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
-
-	"github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"
 )
 
 func TestGoToolLocation(t *testing.T) {
-	t.Skip("skipping test that requires go command")
 	testenv.MustHaveGoBuild(t)
 
 	var exeSuffix string
@@ -53,3 +53,33 @@ func TestGoToolLocation(t *testing.T) {
 		t.Fatalf("%q is not the same file as %q", absWant, goTool)
 	}
 }
+
+func TestHasGoBuild(t *testing.T) {
+	// Removed by Hugo.
+}
+
+func TestMustHaveExec(t *testing.T) {
+	hasExec := false
+	t.Run("MustHaveExec", func(t *testing.T) {
+		testenv.MustHaveExec(t)
+		t.Logf("MustHaveExec did not skip")
+		hasExec = true
+	})
+
+	switch runtime.GOOS {
+	case "js", "wasip1":
+		if hasExec {
+			// js and wasip1 lack an “exec” syscall.
+			t.Errorf("expected MustHaveExec to skip on %v", runtime.GOOS)
+		}
+	case "ios":
+		if b := testenv.Builder(); strings.HasSuffix(b, "-corellium") && !hasExec {
+			// Most ios environments can't exec, but the corellium builder can.
+			t.Errorf("expected MustHaveExec not to skip on %v", b)
+		}
+	default:
+		if b := testenv.Builder(); b != "" && !hasExec {
+			t.Errorf("expected MustHaveExec not to skip on %v", b)
+		}
+	}
+}

tpl/internal/go_templates/testenv/testenv_unix.go

@@ -6,8 +6,15 @@
 
 package testenv
 
-import "syscall"
+import (
+	"syscall"
+)
 
 // Sigquit is the signal to send to kill a hanging subprocess.
 // Send SIGQUIT to get a stack trace.
 var Sigquit = syscall.SIGQUIT
+
+func syscallIsNotSupported(err error) bool {
+	// Removed by Hugo (not supported in Go 1.20)
+	return false
+}