From 3c8c9e1e443a0a49ae42836b82fb1290d42b7857 Mon Sep 17 00:00:00 2001
From: Robert Findley <rfindley@google.com>
Date: Tue, 31 Aug 2021 17:59:05 -0400
Subject: [PATCH] go/types: don't print instance markers for type hashes

This is a port of CL 345891 to go/types.

Change-Id: I5abcb9c9c5110923a743f0c47d9b34b2baabab68
Reviewed-on: https://go-review.googlesource.com/c/go/+/346555
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
Reviewed-by: Robert Griesemer <gri@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
---
 src/go/types/subst.go      | 19 +++++++------------
 src/go/types/typestring.go |  6 +++++-
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/go/types/subst.go b/src/go/types/subst.go
index 4be489e266..f1bdbc34bd 100644
--- a/src/go/types/subst.go
+++ b/src/go/types/subst.go
@@ -259,28 +259,23 @@ func (subst *subster) typ(typ Type) Type {
 var instanceHashing = 0
 
 func instantiatedHash(typ *Named, targs []Type) string {
+	var buf bytes.Buffer
+
 	assert(instanceHashing == 0)
 	instanceHashing++
-	var buf bytes.Buffer
 	w := newTypeWriter(&buf, nil)
 	w.typeName(typ.obj)
 	w.typeList(targs)
 	instanceHashing--
 
-	// With respect to the represented type, whether a
-	// type is fully expanded or stored as instance
-	// does not matter - they are the same types.
-	// Remove the instanceMarkers printed for instances.
-	res := buf.Bytes()
-	i := 0
-	for _, b := range res {
-		if b != instanceMarker {
-			res[i] = b
-			i++
+	if debug {
+		// there should be no instance markers in type hashes
+		for _, b := range buf.Bytes() {
+			assert(b != instanceMarker)
 		}
 	}
 
-	return string(res[:i])
+	return buf.String()
 }
 
 // typOrNil is like typ but if the argument is nil it is replaced with Typ[Invalid].
diff --git a/src/go/types/typestring.go b/src/go/types/typestring.go
index 45ed632335..46e749c84a 100644
--- a/src/go/types/typestring.go
+++ b/src/go/types/typestring.go
@@ -204,7 +204,11 @@ func (w *typeWriter) typ(typ Type) {
 		}
 
 	case *Named:
-		if t.instPos != nil {
+		// Instance markers indicate unexpanded instantiated
+		// types. Write them to aid debugging, but don't write
+		// them when we need an instance hash: whether a type
+		// is fully expanded or not doesn't matter for identity.
+		if instanceHashing == 0 && t.instPos != nil {
 			w.byte(instanceMarker)
 		}
 		w.typeName(t.obj)