cache server-hostname (#176)

Reviewed-on: https://git.ptzo.gdn/feditools/relay/pulls/176
Co-authored-by: Tyr Mactire <tyr@pettingzoo.co>
Co-committed-by: Tyr Mactire <tyr@pettingzoo.co>

go.mod

@@ -3,7 +3,7 @@ module git.ptzo.gdn/feditools/relay
 go 1.19
 
 require (
-	git.ptzo.gdn/feditools/go-lib v0.19.1
+	git.ptzo.gdn/feditools/go-lib v0.20.1
 	github.com/allegro/bigcache/v3 v3.1.0
 	github.com/contribsys/faktory v1.6.2
 	github.com/contribsys/faktory_worker_go v1.6.0

go.sum

@@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.ptzo.gdn/feditools/go-lib v0.19.1 h1:tX4jo4EG9SIciniUkBuLYcTrWckqjYfPToNMFvmSnlg=
-git.ptzo.gdn/feditools/go-lib v0.19.1/go.mod h1:AjT/r7wnZzZqU//GEQ1YGwepx1Pt7BIILTG4edoY+Uw=
+git.ptzo.gdn/feditools/go-lib v0.20.1 h1:UaExoZx2JCUsUENuteY1/RcHubtrfyZdx4ysl1ukpzg=
+git.ptzo.gdn/feditools/go-lib v0.20.1/go.mod h1:AjT/r7wnZzZqU//GEQ1YGwepx1Pt7BIILTG4edoY+Uw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

internal/db/bun/migrations/20221230161426_new.go

@@ -0,0 +1,80 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"git.ptzo.gdn/feditools/relay/internal/models"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/dialect/sqltype"
+	"strings"
+)
+
+func init() {
+	l := logger.WithField("migration", "20221230161426")
+
+	dropColumns := columnList{
+		{
+			Model: (*models.Instance)(nil),
+			Name:  "server_hostname",
+			Type:  sqltype.VarChar,
+		},
+	}
+
+	up := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			for _, c := range dropColumns {
+				query := tx.NewDropColumn().
+					Model(c.Model).
+					ColumnExpr(fmt.Sprintf("%s", c.Name))
+
+				l.Infof(DroppingColumn, c.Name, query.GetTableName())
+
+				_, err := query.Exec(ctx)
+				if err != nil {
+					l.Errorf(DroppingColumnErr, c.Name, query.GetTableName(), err.Error())
+					return err
+				}
+			}
+
+			return nil
+		})
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			for _, c := range dropColumns {
+				var sb strings.Builder
+				sb.WriteString(fmt.Sprintf("%s %s", c.Name, c.Type))
+				if c.NotNull {
+					sb.WriteString(" NOT NULL")
+				}
+				if c.Default != nil {
+					sb.WriteString(" DEFAULT ?")
+				}
+
+				query := tx.NewAddColumn().
+					Model(c.Model)
+
+				if c.Default != nil {
+					query = query.ColumnExpr(sb.String(), c.Default)
+				} else {
+					query = query.ColumnExpr(sb.String())
+				}
+
+				l.Infof(CreatingColumn, c.Name, query.GetTableName())
+
+				_, err := query.Exec(ctx)
+				if err != nil {
+					l.Errorf(CreatingColumnErr, c.Name, query.GetTableName(), err.Error())
+					return err
+				}
+			}
+
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/logic/logic1/instance.go

@@ -53,7 +53,7 @@ func (l *Logic) GetPeers(ctx context.Context) (*[]string, error) {
 	// populate peer list
 	newPeers := make([]string, len(instances))
 	for i, instance := range instances {
-		newPeers[i] = instance.ServerHostname
+		newPeers[i] = instance.Domain
 	}
 
 	// update cache

internal/logic/logic1/instance_self.go

@@ -24,10 +24,9 @@ func (l *Logic) createInstanceSelf(ctx context.Context) (*models.Instance, error
 
 	// create new instance
 	newInstance := &models.Instance{
-		Domain:         l.domain,
-		ServerHostname: l.domain,
-		ActorIRI:       genActorSelf(l.domain),
-		InboxIRI:       path.GenInbox(l.domain).String(),
+		Domain:   l.domain,
+		ActorIRI: genActorSelf(l.domain),
+		InboxIRI: path.GenInbox(l.domain).String(),
 
 		PublicKey: &privatekey.PublicKey,
 	}

internal/models/instance.go

@@ -10,7 +10,6 @@ import (
 	"git.ptzo.gdn/feditools/relay/internal/util"
 	"github.com/sirupsen/logrus"
 	"io"
-	"strings"
 	"time"
 )
 
@@ -20,7 +19,6 @@ type Instance struct {
 	CreatedAt         time.Time      `validate:"-" bun:",nullzero,notnull,default:current_timestamp"`
 	UpdatedAt         time.Time      `validate:"-" bun:",nullzero,notnull,default:current_timestamp"`
 	Domain            string         `validate:"required,fqdn" bun:",nullzero,notnull,unique"`
-	ServerHostname    string         `validate:"required,fqdn" bun:",nullzero,notnull,unique"`
 	Software          string         `validate:"-" bun:",nullzero"`
 	PublicKey         *rsa.PublicKey `validate:"-"`
 	PrivateKey        string         `validate:"-" bun:",nullzero"`
@@ -98,14 +96,7 @@ func (i *Instance) PublicKeyPEM() (string, error) {
 
 // String returns a unicode representation of the domain.
 func (i *Instance) String() string {
-	var sb strings.Builder
-	sb.WriteString(i.UnicodeDomain())
-
-	if i.IsPunycode() || i.Domain != i.ServerHostname {
-		sb.WriteString(" (" + i.ServerHostname + ")")
-	}
-
-	return sb.String()
+	return i.UnicodeDomain()
 }
 
 // UnicodeDomain returns a unicode representation of the domain.

internal/models/instance_feditools.go

@@ -22,10 +22,6 @@ func (i *Instance) GetOAuthClientID() (clientID string) {
 	return i.OAuthClientID
 }
 
-func (i *Instance) GetServerHostname() (hostname string) {
-	return i.ServerHostname
-}
-
 func (i *Instance) GetSoftware() (software string) {
 	return i.Software
 }
@@ -46,10 +42,6 @@ func (i *Instance) SetOAuthClientID(clientID string) {
 	i.OAuthClientID = clientID
 }
 
-func (i *Instance) SetServerHostname(hostname string) {
-	i.ServerHostname = hostname
-}
-
 func (i *Instance) SetSoftware(software string) {
 	i.Software = software
 }

vendor/git.ptzo.gdn/feditools/go-lib/Jenkinsfile

@@ -23,7 +23,6 @@ pipeline {
             string(credentialsId: 'codecov-feditools-go-lib', variable: 'CODECOV_TOKEN')
           ]) {
             sh """#!/bin/bash
-            go get -t -v ./...
             go test -race -coverprofile=coverage.txt -covermode=atomic ./...
             RESULT=\$?
             bash <(curl -s https://codecov.io/bash)

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/account.go

@@ -52,8 +52,17 @@ func (f *FediHelper) GetOrCreateAccount(ctx context.Context, instanceAccountID,
 		return account, nil
 	}
 
+	// get server hostname
+	serverHostname, err := f.GetInstanceHostname(ctx, instance.GetDomain())
+	if err != nil {
+		fhErr := NewErrorf("get server hostname: %s", err.Error())
+		l.Error(fhErr.Error())
+
+		return nil, fhErr
+	}
+
 	// do webfinger
-	webFinger, err := f.FetchWellknownWebFinger(ctx, instance.GetServerHostname(), username, instance.GetDomain())
+	webFinger, err := f.FetchWellknownWebFinger(ctx, serverHostname, username, instance.GetDomain())
 	if err != nil {
 		fhErr := NewErrorf("webfinger %s@%s: %s", username, instance.GetDomain(), err.Error())
 		l.Debug(fhErr.Error())

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/actor.go

@@ -34,13 +34,7 @@ func (a *Actor) MakeInstance(instance Instance) error {
 		return NewErrorf("actor is not type %s", TypeApplication)
 	}
 
-	actorID, err := url.Parse(a.ID)
-	if a.Type != TypeApplication {
-		return NewErrorf("can't parse id: %s", err.Error())
-	}
-
 	instance.SetDomain(a.PreferredUsername)
-	instance.SetServerHostname(actorID.Host)
 	instance.SetActorURI(a.ID)
 	instance.SetInboxURI(a.Endpoints.SharedInbox)
 

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/host_meta.go

@@ -3,7 +3,6 @@ package fedihelper
 import (
 	"context"
 	"encoding/xml"
-	"fmt"
 	"net/url"
 	"strings"
 )
@@ -28,13 +27,26 @@ func (f *FediHelper) FetchHostMeta(ctx context.Context, domain string) (*HostMet
 	defer span.End()
 
 	log := logger.WithField("func", "FetchHostMeta")
-	hostMetaURI := &url.URL{
-		Scheme: "https",
-		Host:   domain,
-		Path:   "/.well-known/host-meta",
-	}
 
-	v, err, _ := f.requestGroup.Do(fmt.Sprintf("hostmeta-%s", hostMetaURI.String()), func() (interface{}, error) {
+	v, err, _ := f.requestGroup.Do("hostmeta-"+domain, func() (interface{}, error) {
+		hostMetaURI := &url.URL{
+			Scheme: "https",
+			Host:   domain,
+			Path:   "/.well-known/host-meta",
+		}
+
+		// check cache
+		cache, err := f.kv.GetHostMeta(tctx, domain)
+		if err != nil && err.Error() != "nil" {
+			fhErr := NewErrorf("redis get: %s", err.Error())
+			log.Error(fhErr.Error())
+
+			return nil, fhErr
+		}
+		if err == nil {
+			return unmarshalHostMeta(cache)
+		}
+
 		// do request
 		bodyBytes, err := f.http.InstanceGet(tctx, hostMetaURI)
 		if err != nil {
@@ -43,15 +55,16 @@ func (f *FediHelper) FetchHostMeta(ctx context.Context, domain string) (*HostMet
 			return nil, err
 		}
 
-		hostMeta := new(HostMeta)
-		err = xml.Unmarshal(bodyBytes, hostMeta)
+		// write cache
+		err = f.kv.SetHostMeta(tctx, domain, bodyBytes, f.actorCacheExp)
 		if err != nil {
-			log.Errorf("decode xml: %s", err.Error())
+			fhErr := NewErrorf("redis set: %s", err.Error())
+			log.Error(fhErr.Error())
 
-			return nil, err
+			return nil, fhErr
 		}
 
-		return hostMeta, nil
+		return unmarshalHostMeta(bodyBytes)
 	})
 
 	if err != nil {
@@ -91,3 +104,13 @@ func (*FediHelper) WebfingerURIFromHostMeta(hostMeta *HostMeta) (string, error)
 
 	return hostMetaURIFTemplate, nil
 }
+
+func unmarshalHostMeta(body []byte) (*HostMeta, error) {
+	hostMeta := new(HostMeta)
+	err := xml.Unmarshal(body, hostMeta)
+	if err != nil {
+		return nil, NewErrorf("unmarshal: %s", err.Error())
+	}
+
+	return hostMeta, nil
+}

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/instance.go

@@ -12,7 +12,6 @@ type Instance interface {
 	GetOAuthClientSecret() (clientSecret string, err error)
 	GetDomain() (domain string)
 	GetID() (id int64)
-	GetServerHostname() (hostname string)
 	GetSoftware() (software string)
 
 	SetActorURI(actorURI string)
@@ -20,7 +19,6 @@ type Instance interface {
 	SetOAuthClientSecret(clientSecret string) error
 	SetDomain(domain string)
 	SetInboxURI(inboxURI string)
-	SetServerHostname(hostname string)
 	SetSoftware(software string)
 
 	// Instance Info
@@ -98,12 +96,35 @@ func (f *FediHelper) GenerateFediInstanceFromDomain(ctx context.Context, domain
 	instance.SetActorURI(actorURI.String())
 	instance.SetInboxURI(actor.Endpoints.SharedInbox)
 	instance.SetDomain(domain)
-	instance.SetServerHostname(hostMetaURI.Host)
 	instance.SetSoftware(nodeinfo.Software.Name)
 
 	return nil
 }
 
+func (f *FediHelper) GetInstanceHostname(ctx context.Context, domain string) (string, error) {
+	ctx, span := f.tracer.Start(ctx, "GetInstanceHostname", f.tracerAttrs...)
+	defer span.End()
+
+	// pull host meta
+	hostMeta, err := f.FetchHostMeta(ctx, domain)
+	if err != nil {
+		return "", NewErrorf("host meta: %s", err.Error())
+	}
+
+	// perform web finger
+	webfingerURI := hostMeta.WebfingerURI()
+	if webfingerURI == "" {
+		return "", NewError("host meta missing webfinger URI")
+	}
+
+	hostMetaURI, err := url.Parse(webfingerURI.String())
+	if err != nil {
+		return "", NewErrorf("parsing host meta uri: %s", err.Error())
+	}
+
+	return hostMetaURI.Host, nil
+}
+
 func (f *FediHelper) instanceWebFinger(ctx context.Context, software, host, domain string) (*WebFinger, error) {
 	switch software {
 	case string(SoftwarePleroma):

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/instance_info.go

@@ -20,7 +20,16 @@ func (f *FediHelper) UpdateInstanceInfo(ctx context.Context, instance Instance)
 	l := logger.WithField("func", "UpdateInstanceInfo")
 
 	if instance.GetSoftware() == "" {
-		nodeinfo, err := f.getNodeInfo20ForDomain(ctx, instance.GetServerHostname())
+		// get server hostname
+		serverHostname, err := f.GetInstanceHostname(ctx, instance.GetDomain())
+		if err != nil {
+			fhErr := NewErrorf("get server hostname: %s", err.Error())
+			l.Error(fhErr.Error())
+
+			return fhErr
+		}
+
+		nodeinfo, err := f.getNodeInfo20ForDomain(ctx, serverHostname)
 		if err != nil {
 			fhErr := NewErrorf("get nodeinfo 2.0: %s", err.Error())
 			l.Debug(fhErr.Error())

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/login.go

@@ -14,7 +14,16 @@ func (f *FediHelper) GetLoginURL(ctx context.Context, redirectURI *url.URL, inst
 
 	instanceUpdated := false
 	if instance.GetSoftware() == "" {
-		nodeinfo, err := f.getNodeInfo20ForDomain(ctx, instance.GetServerHostname())
+		// get server hostname
+		serverHostname, err := f.GetInstanceHostname(ctx, instance.GetDomain())
+		if err != nil {
+			fhErr := NewErrorf("get server hostname: %s", err.Error())
+			l.Error(fhErr.Error())
+
+			return nil, false, fhErr
+		}
+
+		nodeinfo, err := f.getNodeInfo20ForDomain(ctx, serverHostname)
 		if err != nil {
 			fhErr := NewErrorf("get nodeinfo 2.0: %s", err.Error())
 			l.Error(fhErr.Error())

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/mastodon/login.go

@@ -2,6 +2,7 @@ package mastodon
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 
 	"git.ptzo.gdn/feditools/go-lib/fedihelper"
@@ -25,10 +26,17 @@ func (h *Helper) GetAccessToken(ctx context.Context, redirectURI *url.URL, insta
 }
 
 // MakeLoginURI creates a login redirect url for mastodon.
-func (h *Helper) MakeLoginURI(_ context.Context, redirectURI *url.URL, instance fedihelper.Instance) (*url.URL, error) {
+func (h *Helper) MakeLoginURI(ctx context.Context, redirectURI *url.URL, instance fedihelper.Instance) (*url.URL, error) {
+	serverHostname, err := h.fedi.GetInstanceHostname(ctx, instance.GetDomain())
+	if err != nil {
+		fhErr := fmt.Errorf("get server hostname: %s", err.Error())
+
+		return nil, fhErr
+	}
+
 	u := &url.URL{
 		Scheme: "https",
-		Host:   instance.GetServerHostname(),
+		Host:   serverHostname,
 		Path:   "/oauth/authorize",
 	}
 	q := u.Query()

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/mastodon/mastodon.go

@@ -2,6 +2,7 @@ package mastodon
 
 import (
 	"context"
+	"fmt"
 
 	"git.ptzo.gdn/feditools/go-lib/fedihelper"
 	mastodon "github.com/mattn/go-mastodon"
@@ -36,9 +37,16 @@ func New(k fedihelper.KV, t *fedihelper.Transport, appClientName, appWebsite, ex
 }
 
 // newClient return new mastodon API client.
-func (h *Helper) newClient(_ context.Context, instance fedihelper.Instance, accessToken string) (*mastodon.Client, error) {
+func (h *Helper) newClient(ctx context.Context, instance fedihelper.Instance, accessToken string) (*mastodon.Client, error) {
 	l := logger.WithField("func", "newClient")
 
+	serverHostname, err := h.fedi.GetInstanceHostname(ctx, instance.GetDomain())
+	if err != nil {
+		fhErr := fmt.Errorf("get server hostname: %s", err.Error())
+
+		return nil, fhErr
+	}
+
 	// get client secret
 	clientSecret, err := instance.GetOAuthClientSecret()
 	if err != nil {
@@ -49,7 +57,7 @@ func (h *Helper) newClient(_ context.Context, instance fedihelper.Instance, acce
 
 	// create client
 	client := mastodon.NewClient(&mastodon.Config{
-		Server:       "https://" + instance.GetServerHostname(),
+		Server:       "https://" + serverHostname,
 		ClientID:     instance.GetOAuthClientID(),
 		ClientSecret: clientSecret,
 		AccessToken:  accessToken,
@@ -62,10 +70,17 @@ func (h *Helper) newClient(_ context.Context, instance fedihelper.Instance, acce
 }
 
 // newClient return new mastodon API client.
-func (h *Helper) newUnauthClient(_ context.Context, instance fedihelper.Instance) (*mastodon.Client, error) {
+func (h *Helper) newUnauthClient(ctx context.Context, instance fedihelper.Instance) (*mastodon.Client, error) {
+	serverHostname, err := h.fedi.GetInstanceHostname(ctx, instance.GetDomain())
+	if err != nil {
+		fhErr := fmt.Errorf("get server hostname: %s", err.Error())
+
+		return nil, fhErr
+	}
+
 	// create client
 	client := mastodon.NewClient(&mastodon.Config{
-		Server: "https://" + instance.GetServerHostname(),
+		Server: "https://" + serverHostname,
 	})
 
 	// apply custom transport

vendor/git.ptzo.gdn/feditools/go-lib/fedihelper/mastodon/register.go

@@ -2,6 +2,7 @@ package mastodon
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/url"
 
@@ -14,11 +15,18 @@ import (
 func (h *Helper) RegisterApp(ctx context.Context, redirectURI *url.URL, instance fedihelper.Instance) (clientID string, clientSecret string, err error) {
 	l := logger.WithField("func", "RegisterApp")
 	v, serr, _ := h.registerAppGroup.Do(instance.GetDomain(), func() (interface{}, error) {
+		serverHostname, err := h.fedi.GetInstanceHostname(ctx, instance.GetDomain())
+		if err != nil {
+			fhErr := fmt.Errorf("get server hostname: %s", err.Error())
+
+			return nil, fhErr
+		}
+
 		app, merr := mastodon.RegisterApp(ctx, &mastodon.AppConfig{
 			Client: http.Client{
 				Transport: h.transport.Client().Transport(),
 			},
-			Server:       "https://" + instance.GetServerHostname(),
+			Server:       "https://" + serverHostname,
 			ClientName:   h.appClientName,
 			Scopes:       "read:accounts",
 			Website:      h.appWebsite,

vendor/modules.txt

@@ -1,4 +1,4 @@
-# git.ptzo.gdn/feditools/go-lib v0.19.1
+# git.ptzo.gdn/feditools/go-lib v0.20.1
 ## explicit; go 1.17
 git.ptzo.gdn/feditools/go-lib
 git.ptzo.gdn/feditools/go-lib/database