cmd/compile: catch pointless recursion on function types

If a function type has no type parameters, note when it
is visited and do not recur.  (It must be visited
at least once because of closures and their associated
types occurring in a generic context).

Fixes #51832.

Change-Id: Iee20612ffd0a03b838b9e59615f4a0206fc8940b
Reviewed-on: https://go-review.googlesource.com/c/go/+/406714
Reviewed-by: Keith Randall <khr@google.com>

src/cmd/compile/internal/typecheck/subr.go

@@ -1013,6 +1013,8 @@ type Tsubster struct {
 	Vars map[*ir.Name]*ir.Name
 	// If non-nil, function to substitute an incomplete (TFORW) type.
 	SubstForwFunc func(*types.Type) *types.Type
+	// Prevent endless recursion on functions. See #51832.
+	Funcs map[*types.Type]bool
 }
 
 // Typ computes the type obtained by substituting any type parameter or shape in t
@@ -1030,7 +1032,8 @@ func (ts *Tsubster) Typ(t *types.Type) *types.Type {
 }
 
 func (ts *Tsubster) typ1(t *types.Type) *types.Type {
-	if !t.HasTParam() && !t.HasShape() && t.Kind() != types.TFUNC {
+	hasParamOrShape := t.HasTParam() || t.HasShape()
+	if !hasParamOrShape && t.Kind() != types.TFUNC {
 		// Note: function types need to be copied regardless, as the
 		// types of closures may contain declarations that need
 		// to be copied. See #45738.
@@ -1066,10 +1069,10 @@ func (ts *Tsubster) typ1(t *types.Type) *types.Type {
 
 	var newsym *types.Sym
 	var neededTargs []*types.Type
-	var targsChanged bool
+	var targsChanged bool // == are there any substitutions from this
 	var forw *types.Type
 
-	if t.Sym() != nil && (t.HasTParam() || t.HasShape()) {
+	if t.Sym() != nil && hasParamOrShape {
 		// Need to test for t.HasTParam() again because of special TFUNC case above.
 		// Translate the type params for this type according to
 		// the tparam/targs mapping from subst.
@@ -1144,6 +1147,17 @@ func (ts *Tsubster) typ1(t *types.Type) *types.Type {
 		}
 
 	case types.TFUNC:
+		// watch out for endless recursion on plain function types that mention themselves, e.g. "type T func() T"
+		if !hasParamOrShape {
+			if ts.Funcs[t] { // Visit such function types only once.
+				return t
+			}
+			if ts.Funcs == nil {
+				// allocate lazily
+				ts.Funcs = make(map[*types.Type]bool)
+			}
+			ts.Funcs[t] = true
+		}
 		newrecvs := ts.tstruct(t.Recvs(), false)
 		newparams := ts.tstruct(t.Params(), false)
 		newresults := ts.tstruct(t.Results(), false)
@@ -1179,6 +1193,9 @@ func (ts *Tsubster) typ1(t *types.Type) *types.Type {
 			newt = types.NewSignature(t.Pkg(), newrecv, tparamfields,
 				newparams.FieldSlice(), newresults.FieldSlice())
 		}
+		if !hasParamOrShape {
+			delete(ts.Funcs, t)
+		}
 
 	case types.TINTER:
 		newt = ts.tinter(t, targsChanged)

test/typeparam/issue51832.go

@@ -0,0 +1,25 @@
+// compile
+
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+type F func() F
+
+func do[T any]() F {
+	return nil
+}
+
+type G[T any] func() G[T]
+
+//go:noinline
+func dog[T any]() G[T] {
+	return nil
+}
+
+func main() {
+	do[int]()
+	dog[int]()
+}