mirror of
https://git.ptzo.gdn/feditools/relay.git
synced 2024-09-21 11:47:12 +00:00
Notifications (#43)
This commit is contained in:
Tyr Mactire
GitHub
parent
3a43518ae6
commit
e4a9aa08ab
99
cmd/relay/action/server/http_server.go
Normal file
99
cmd/relay/action/server/http_server.go
Normal file
@ -0,0 +1,99 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/feditools/go-lib"
|
||||
liblanguage "github.com/feditools/go-lib/language"
|
||||
"github.com/feditools/go-lib/metrics"
|
||||
"github.com/feditools/relay/internal/config"
|
||||
"github.com/feditools/relay/internal/db"
|
||||
"github.com/feditools/relay/internal/fedi"
|
||||
"github.com/feditools/relay/internal/http"
|
||||
"github.com/feditools/relay/internal/http/activitypub"
|
||||
"github.com/feditools/relay/internal/http/static"
|
||||
"github.com/feditools/relay/internal/http/webapp"
|
||||
"github.com/feditools/relay/internal/logic"
|
||||
"github.com/feditools/relay/internal/runner"
|
||||
"github.com/feditools/relay/internal/token"
|
||||
"github.com/go-redis/redis/v8"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func newHttpServer(
|
||||
ctx context.Context,
|
||||
dbClient db.DB,
|
||||
fediMod *fedi.Module,
|
||||
languageMod *liblanguage.Module,
|
||||
logicMod logic.Logic,
|
||||
metricsCollector metrics.Collector,
|
||||
redisClient redis.UniversalClient,
|
||||
runnerMod runner.Runner,
|
||||
tokz *token.Tokenizer,
|
||||
) (*http.Server, error) {
|
||||
l := logger.WithField("func", "newHttpServer")
|
||||
|
||||
// create http server
|
||||
l.Debug("creating http server")
|
||||
server, err := http.NewServer(ctx, metricsCollector)
|
||||
if err != nil {
|
||||
l.Errorf("http server: %s", err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// create web modules
|
||||
var webModules []http.Module
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleActivityPub) {
|
||||
l.Infof("adding %s module", config.ServerRoleActivityPub)
|
||||
apMod, err := activitypub.New(ctx, logicMod, runnerMod)
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleActivityPub, err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleStatic) {
|
||||
l.Infof("adding %s module", config.ServerRoleStatic)
|
||||
apMod, err := static.New()
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleStatic, err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleWebapp) {
|
||||
l.Infof("adding %s module", config.ServerRoleWebapp)
|
||||
apMod, err := webapp.New(
|
||||
ctx,
|
||||
dbClient,
|
||||
fediMod,
|
||||
languageMod,
|
||||
logicMod,
|
||||
metricsCollector,
|
||||
redisClient,
|
||||
runnerMod,
|
||||
tokz,
|
||||
)
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleWebapp, err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
|
||||
// add modules to servers
|
||||
for _, mod := range webModules {
|
||||
mod.SetServer(server)
|
||||
err := mod.Route(server)
|
||||
if err != nil {
|
||||
l.Errorf("loading %s module: %s", mod.Name(), err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return server, nil
|
||||
}
|
||||
30
cmd/relay/action/server/notifier.go
Normal file
30
cmd/relay/action/server/notifier.go
Normal file
@ -0,0 +1,30 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/logic"
|
||||
"github.com/feditools/relay/internal/notification/manager"
|
||||
"github.com/feditools/relay/internal/notification/telegram"
|
||||
)
|
||||
|
||||
func newNotifier(
|
||||
logicMod logic.Logic,
|
||||
) (*manager.Manager, error) {
|
||||
l := logger.WithField("func", "newNotifier")
|
||||
|
||||
newManager, err := manager.New()
|
||||
if err != nil {
|
||||
l.Errorf("notification manager: %s", err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
telegramMod, err := telegram.New(logicMod)
|
||||
if err != nil {
|
||||
l.Errorf("notification telegram: %s", err.Error())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
newManager.AddService(telegramMod)
|
||||
|
||||
return newManager, nil
|
||||
}
|
||||
@ -3,7 +3,6 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/feditools/go-lib"
|
||||
"github.com/feditools/go-lib/language"
|
||||
"github.com/feditools/go-lib/metrics/statsd"
|
||||
"github.com/feditools/relay/cmd/relay/action"
|
||||
@ -12,9 +11,6 @@ import (
|
||||
"github.com/feditools/relay/internal/db/bun"
|
||||
"github.com/feditools/relay/internal/fedi"
|
||||
"github.com/feditools/relay/internal/http"
|
||||
"github.com/feditools/relay/internal/http/activitypub"
|
||||
"github.com/feditools/relay/internal/http/static"
|
||||
"github.com/feditools/relay/internal/http/webapp"
|
||||
"github.com/feditools/relay/internal/kv/redis"
|
||||
"github.com/feditools/relay/internal/logic/logic1"
|
||||
"github.com/feditools/relay/internal/runner/faktory"
|
||||
@ -133,6 +129,16 @@ var Start action.Action = func(topCtx context.Context) error {
|
||||
}
|
||||
logicMod.SetFedi(fediMod)
|
||||
|
||||
// create language module
|
||||
notifier, err := newNotifier(logicMod)
|
||||
if err != nil {
|
||||
l.Errorf("notifier: %s", err.Error())
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
logicMod.SetNotifier(notifier)
|
||||
|
||||
// create runner
|
||||
runnerMod, err := faktory.New(logicMod)
|
||||
if err != nil {
|
||||
@ -146,7 +152,17 @@ var Start action.Action = func(topCtx context.Context) error {
|
||||
|
||||
// create http server
|
||||
l.Debug("creating http server")
|
||||
server, err := http.NewServer(ctx, metricsCollector)
|
||||
server, err := newHttpServer(
|
||||
ctx,
|
||||
dbClient,
|
||||
fediMod,
|
||||
languageMod,
|
||||
logicMod,
|
||||
metricsCollector,
|
||||
kvClient.RedisClient(),
|
||||
runnerMod,
|
||||
tokz,
|
||||
)
|
||||
if err != nil {
|
||||
l.Errorf("http server: %s", err.Error())
|
||||
cancel()
|
||||
@ -154,64 +170,6 @@ var Start action.Action = func(topCtx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// create web modules
|
||||
var webModules []http.Module
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleActivityPub) {
|
||||
l.Infof("adding %s module", config.ServerRoleActivityPub)
|
||||
apMod, err := activitypub.New(ctx, logicMod, runnerMod)
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleActivityPub, err.Error())
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleStatic) {
|
||||
l.Infof("adding %s module", config.ServerRoleStatic)
|
||||
apMod, err := static.New()
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleStatic, err.Error())
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
if lib.ContainsString(viper.GetStringSlice(config.Keys.ServerRoles), config.ServerRoleWebapp) {
|
||||
l.Infof("adding %s module", config.ServerRoleWebapp)
|
||||
apMod, err := webapp.New(
|
||||
ctx,
|
||||
dbClient,
|
||||
fediMod,
|
||||
languageMod,
|
||||
logicMod,
|
||||
metricsCollector,
|
||||
kvClient.RedisClient(),
|
||||
runnerMod,
|
||||
tokz,
|
||||
)
|
||||
if err != nil {
|
||||
l.Errorf("%s: %s", config.ServerRoleWebapp, err.Error())
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
webModules = append(webModules, apMod)
|
||||
}
|
||||
|
||||
// add modules to servers
|
||||
for _, mod := range webModules {
|
||||
mod.SetServer(server)
|
||||
err := mod.Route(server)
|
||||
if err != nil {
|
||||
l.Errorf("loading %s module: %s", mod.Name(), err.Error())
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// ** start application **
|
||||
errChan := make(chan error)
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ func Global(cmd *cobra.Command, values config.Values) {
|
||||
// application
|
||||
cmd.PersistentFlags().Int(config.Keys.ActorKeySize, values.ActorKeySize, usage.ActorKeySize)
|
||||
cmd.PersistentFlags().String(config.Keys.ApplicationName, values.ApplicationName, usage.ApplicationName)
|
||||
cmd.PersistentFlags().String(config.Keys.ApplicationWebsite, values.ApplicationWebsite, usage.ApplicationWebsite)
|
||||
cmd.PersistentFlags().Int(config.Keys.CachedActivityLimit, values.CachedActivityLimit, usage.CachedActivityLimit)
|
||||
cmd.PersistentFlags().Int(config.Keys.CachedActorLimit, values.CachedActorLimit, usage.CachedActorLimit)
|
||||
cmd.PersistentFlags().Int(config.Keys.CachedDigestLimit, values.CachedDigestLimit, usage.CachedDigestLimit)
|
||||
|
||||
@ -22,9 +22,9 @@ func main() {
|
||||
|
||||
var v string
|
||||
if len(Commit) < 7 {
|
||||
v = Version
|
||||
v = "v" + Version
|
||||
} else {
|
||||
v = Version + " " + Commit[:7]
|
||||
v = "v" + Version + "-" + Commit[:7]
|
||||
}
|
||||
|
||||
// set software version
|
||||
|
||||
12
go.mod
12
go.mod
@ -5,7 +5,7 @@ go 1.18
|
||||
require (
|
||||
github.com/contribsys/faktory v1.6.1
|
||||
github.com/contribsys/faktory_worker_go v1.6.0
|
||||
github.com/feditools/go-lib v0.14.0
|
||||
github.com/feditools/go-lib v0.15.0
|
||||
github.com/go-fed/activity v1.0.0
|
||||
github.com/go-fed/httpsig v1.1.0
|
||||
github.com/go-http-utils/etag v0.0.0-20161124023236-513ea8f21eb1
|
||||
@ -18,6 +18,7 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/jackc/pgconn v1.12.1
|
||||
github.com/jackc/pgx/v4 v4.16.1
|
||||
github.com/nickname76/telegrambot v1.1.1
|
||||
github.com/rbcervilla/redisstore/v8 v8.1.0
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/speps/go-hashids/v2 v2.0.1
|
||||
@ -35,6 +36,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/cactus/go-statsd-client/v5 v5.0.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
@ -56,13 +58,18 @@ require (
|
||||
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
|
||||
github.com/jackc/pgtype v1.11.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.6 // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mattn/go-mastodon v0.0.5-0.20211214115546-7745e19ff787 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nickname76/repeater v1.0.1 // indirect
|
||||
github.com/nicksnyder/go-i18n/v2 v2.2.0 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
|
||||
@ -73,12 +80,13 @@ require (
|
||||
github.com/subosito/gotenv v1.3.0 // indirect
|
||||
github.com/tdewolff/parse/v2 v2.6.1 // indirect
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.38.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.10 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
|
||||
|
||||
30
go.sum
30
go.sum
@ -43,6 +43,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
||||
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/cactus/go-statsd-client/v5 v5.0.0 h1:KqvIQtc9qt34uq+nu4nd1PwingWfBt/IISgtUQ2nSJk=
|
||||
github.com/cactus/go-statsd-client/v5 v5.0.0/go.mod h1:COEvJ1E+/E2L4q6QE5CkjWPi4eeDw9maJBMIuMPBZbY=
|
||||
@ -88,8 +90,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/feditools/go-lib v0.14.0 h1:rCkH5mKmbPof5F/LByTt3SLFKx07kgkJV0H0cw+6iH8=
|
||||
github.com/feditools/go-lib v0.14.0/go.mod h1:LtdLBvApYAhdblS6rTGQ12ZzySI9/6PyTiZlxw5uX10=
|
||||
github.com/feditools/go-lib v0.15.0 h1:B1H/Z+nPOYtsXdxrUGcXIrS0QqXQJht1hKPWtRaQBtE=
|
||||
github.com/feditools/go-lib v0.15.0/go.mod h1:LtdLBvApYAhdblS6rTGQ12ZzySI9/6PyTiZlxw5uX10=
|
||||
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
|
||||
github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
@ -169,6 +171,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
@ -261,11 +264,16 @@ github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
@ -310,6 +318,15 @@ github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4
|
||||
github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/nickname76/repeater v1.0.1 h1:RLACAbPHMwn37XvwwwTtqhZMa/M1nwsodxQ5rmjzh/s=
|
||||
github.com/nickname76/repeater v1.0.1/go.mod h1:oJnqHAjbEamcjVBm8sHaexIVFdicS2YU6cJK0KZhgig=
|
||||
github.com/nickname76/telegrambot v1.1.1 h1:LRIkNFshhdBnNMMrKLrKV5SdUE6/68YvL0NNFtPqR9I=
|
||||
github.com/nickname76/telegrambot v1.1.1/go.mod h1:YmbzBD+A0Wiyu7Er2JP1WDpXigl/ZS09ldnKTKgDfXo=
|
||||
github.com/nicksnyder/go-i18n/v2 v2.2.0 h1:MNXbyPvd141JJqlU6gJKrczThxJy+kdCNivxZpBQFkw=
|
||||
github.com/nicksnyder/go-i18n/v2 v2.2.0/go.mod h1:4OtLfzqyAxsscyCb//3gfqSvBc81gImX91LrZzczN1o=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
@ -403,6 +420,11 @@ github.com/uptrace/bun/dialect/sqlitedialect v1.1.7/go.mod h1:GjqiPWAa9JCLlv51mB
|
||||
github.com/uptrace/bun/extra/bundebug v1.1.7 h1:YbW7i9pUfPJMzclSzdHslIvAAR0WO9dW34ctL1Xh+UM=
|
||||
github.com/uptrace/bun/extra/bundebug v1.1.7/go.mod h1:WoBnTrBG9CXITZUw+UfF+DYjWi71boo8FKZGuS5qDzA=
|
||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.38.0 h1:yTjSSNjuDi2PPvXY2836bIwLmiTS2T4T9p1coQshpco=
|
||||
github.com/valyala/fasthttp v1.38.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
@ -444,6 +466,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -519,6 +542,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -595,6 +619,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
|
||||
|
||||
@ -48,6 +48,10 @@ func (c *Client) CountAccountsForInstance(ctx context.Context, instanceID int64)
|
||||
func (c *Client) CreateAccount(ctx context.Context, account *models.Account) db.Error {
|
||||
metric := c.metrics.NewDBQuery("CreateAccount")
|
||||
|
||||
now := time.Now()
|
||||
account.CreatedAt = now
|
||||
account.UpdatedAt = now
|
||||
|
||||
if err := doCreate(ctx, c.bun, account); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -168,6 +172,9 @@ func (c *Client) ReadAccountsPage(ctx context.Context, index, count int) ([]*mod
|
||||
func (c *Client) UpdateAccount(ctx context.Context, account *models.Account) db.Error {
|
||||
metric := c.metrics.NewDBQuery("UpdateAccount")
|
||||
|
||||
now := time.Now()
|
||||
account.UpdatedAt = now
|
||||
|
||||
if err := doUpdate(ctx, c.bun, account); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"github.com/feditools/relay/internal/db"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CountBlocks returns the number of domain blocks.
|
||||
@ -31,6 +32,10 @@ func (c *Client) CountBlocks(ctx context.Context) (int64, db.Error) {
|
||||
func (c *Client) CreateBlock(ctx context.Context, blocks ...*models.Block) db.Error {
|
||||
metric := c.metrics.NewDBQuery("CreateBlock")
|
||||
|
||||
now := time.Now()
|
||||
setBlocksCreatedAt(&blocks, now)
|
||||
setBlocksUpdatedAt(&blocks, now)
|
||||
|
||||
if err := doCreate(ctx, c.bun, &blocks); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -162,6 +167,9 @@ func (c *Client) ReadBlocksPage(ctx context.Context, index, count int) ([]*model
|
||||
func (c *Client) UpdateBlock(ctx context.Context, blocks ...*models.Block) db.Error {
|
||||
metric := c.metrics.NewDBQuery("UpdateBlock")
|
||||
|
||||
now := time.Now()
|
||||
setBlocksUpdatedAt(&blocks, now)
|
||||
|
||||
if err := doUpdate(ctx, c.bun, &blocks); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -184,3 +192,15 @@ func newBlocksQ(c bun.IDB, blocks *[]*models.Block) *bun.SelectQuery {
|
||||
NewSelect().
|
||||
Model(blocks)
|
||||
}
|
||||
|
||||
func setBlocksCreatedAt(blocks *[]*models.Block, t time.Time) {
|
||||
for _, c := range *blocks {
|
||||
c.CreatedAt = t
|
||||
}
|
||||
}
|
||||
|
||||
func setBlocksUpdatedAt(blocks *[]*models.Block, t time.Time) {
|
||||
for _, c := range *blocks {
|
||||
c.UpdatedAt = t
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,12 +6,17 @@ import (
|
||||
"github.com/feditools/relay/internal/db"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CreateConfig stores the configs.
|
||||
func (c *Client) CreateConfig(ctx context.Context, configs ...*models.Config) db.Error {
|
||||
metric := c.metrics.NewDBQuery("CreateConfig")
|
||||
|
||||
now := time.Now()
|
||||
setConfigsCreatedAt(&configs, now)
|
||||
setConfigsUpdatedAt(&configs, now)
|
||||
|
||||
if err := doCreate(ctx, c.bun, &configs); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -27,6 +32,10 @@ func (c *Client) CreateConfig(ctx context.Context, configs ...*models.Config) db
|
||||
func (c *Client) CreateConfigTX(ctx context.Context, txID db.TxID, configs ...*models.Config) db.Error {
|
||||
metric := c.metrics.NewDBQuery("CreateConfigTX")
|
||||
|
||||
now := time.Now()
|
||||
setConfigsCreatedAt(&configs, now)
|
||||
setConfigsUpdatedAt(&configs, now)
|
||||
|
||||
tx, err := c.getTx(txID)
|
||||
if err != nil {
|
||||
go metric.Done(true)
|
||||
@ -88,7 +97,8 @@ func (c *Client) ReadConfig(ctx context.Context, id int64) (*models.Config, db.E
|
||||
|
||||
config := new(models.Config)
|
||||
err := newConfigQ(c.bun, config).
|
||||
Where("id = ?", id).
|
||||
Relation("Account").
|
||||
Where("config.id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
dbErr := c.bun.ProcessError(err)
|
||||
@ -108,13 +118,69 @@ func (c *Client) ReadConfig(ctx context.Context, id int64) (*models.Config, db.E
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ReadConfigsForKeys returns one config.
|
||||
func (c *Client) ReadConfigsForKeys(ctx context.Context, keys ...models.ConfigKey) ([]*models.Config, db.Error) {
|
||||
metric := c.metrics.NewDBQuery("ReadConfigsForKeys")
|
||||
func (c *Client) ReadConfigsForAccountForKeys(ctx context.Context, accountID int64, keys ...models.ConfigKey) ([]*models.Config, db.Error) {
|
||||
metric := c.metrics.NewDBQuery("ReadConfigsForAccountForKeys")
|
||||
|
||||
var configs []*models.Config
|
||||
err := newConfigsQ(c.bun, &configs).
|
||||
Relation("Account").
|
||||
Where("config.key IN (?)", bun.In(keys)).
|
||||
Where("config.account_id = ?", accountID).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
dbErr := c.bun.ProcessError(err)
|
||||
|
||||
if errors.Is(dbErr, db.ErrNoEntries) {
|
||||
// report no entries as a non error
|
||||
go metric.Done(false)
|
||||
} else {
|
||||
go metric.Done(true)
|
||||
}
|
||||
|
||||
return nil, dbErr
|
||||
}
|
||||
|
||||
go metric.Done(false)
|
||||
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
// ReadConfigsForKeysSystem returns one config.
|
||||
func (c *Client) ReadConfigsForKeysSystem(ctx context.Context, keys ...models.ConfigKey) ([]*models.Config, db.Error) {
|
||||
metric := c.metrics.NewDBQuery("ReadConfigsForKeysSystem")
|
||||
|
||||
var configs []*models.Config
|
||||
err := newConfigsQ(c.bun, &configs).
|
||||
Where("key IN (?)", bun.In(keys)).
|
||||
Where("account_id is NULL").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
dbErr := c.bun.ProcessError(err)
|
||||
|
||||
if errors.Is(dbErr, db.ErrNoEntries) {
|
||||
// report no entries as a non error
|
||||
go metric.Done(false)
|
||||
} else {
|
||||
go metric.Done(true)
|
||||
}
|
||||
|
||||
return nil, dbErr
|
||||
}
|
||||
|
||||
go metric.Done(false)
|
||||
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
// ReadConfigsForKeysUser returns one config.
|
||||
func (c *Client) ReadConfigsForKeysUser(ctx context.Context, keys ...models.ConfigKey) ([]*models.Config, db.Error) {
|
||||
metric := c.metrics.NewDBQuery("ReadConfigsForKeysUser")
|
||||
|
||||
var configs []*models.Config
|
||||
err := newConfigsQ(c.bun, &configs).
|
||||
Relation("Account").
|
||||
Where("config.key IN (?)", bun.In(keys)).
|
||||
Where("config.account_id is not NULL").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
dbErr := c.bun.ProcessError(err)
|
||||
@ -138,7 +204,10 @@ func (c *Client) ReadConfigsForKeys(ctx context.Context, keys ...models.ConfigKe
|
||||
func (c *Client) UpdateConfig(ctx context.Context, configs ...*models.Config) db.Error {
|
||||
metric := c.metrics.NewDBQuery("UpdateConfig")
|
||||
|
||||
if err := doUpdate(ctx, c.bun, &configs); err != nil {
|
||||
now := time.Now()
|
||||
setConfigsUpdatedAt(&configs, now)
|
||||
|
||||
if _, err := newConfigsUpdateQ(c.bun, &configs).Exec(ctx); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
return c.bun.errProc(err)
|
||||
@ -153,6 +222,9 @@ func (c *Client) UpdateConfig(ctx context.Context, configs ...*models.Config) db
|
||||
func (c *Client) UpdateConfigTX(ctx context.Context, txID db.TxID, configs ...*models.Config) db.Error {
|
||||
metric := c.metrics.NewDBQuery("UpdateConfigTX")
|
||||
|
||||
now := time.Now()
|
||||
setConfigsUpdatedAt(&configs, now)
|
||||
|
||||
tx, err := c.getTx(txID)
|
||||
if err != nil {
|
||||
go metric.Done(true)
|
||||
@ -160,7 +232,7 @@ func (c *Client) UpdateConfigTX(ctx context.Context, txID db.TxID, configs ...*m
|
||||
return c.bun.errProc(err)
|
||||
}
|
||||
|
||||
if err := doUpdate(ctx, tx, &configs); err != nil {
|
||||
if _, err := newConfigsUpdateQ(tx, &configs).Exec(ctx); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
return c.bun.errProc(err)
|
||||
@ -182,3 +254,30 @@ func newConfigsQ(c bun.IDB, configs *[]*models.Config) *bun.SelectQuery {
|
||||
NewSelect().
|
||||
Model(configs)
|
||||
}
|
||||
|
||||
func newConfigsUpdateQ(c bun.IDB, configs *[]*models.Config) *bun.UpdateQuery {
|
||||
values := c.NewValues(configs)
|
||||
return c.
|
||||
NewUpdate().
|
||||
With("_data", values).
|
||||
Model((*models.Config)(nil)).
|
||||
TableExpr("_data").
|
||||
Set("created_at = _data.created_at").
|
||||
Set("updated_at = _data.updated_at").
|
||||
Set("key = _data.key").
|
||||
Set("value = _data.value").
|
||||
Set("is_encrypted = _data.is_encrypted").
|
||||
Where("config.id = _data.id")
|
||||
}
|
||||
|
||||
func setConfigsCreatedAt(configs *[]*models.Config, t time.Time) {
|
||||
for _, c := range *configs {
|
||||
c.CreatedAt = t
|
||||
}
|
||||
}
|
||||
|
||||
func setConfigsUpdatedAt(configs *[]*models.Config, t time.Time) {
|
||||
for _, c := range *configs {
|
||||
c.UpdatedAt = t
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,14 +8,14 @@ import (
|
||||
"github.com/feditools/relay/internal/db"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CountInstances returns the number of federated instances.
|
||||
func (c *Client) CountInstances(ctx context.Context) (int64, db.Error) {
|
||||
metric := c.metrics.NewDBQuery("CountInstances")
|
||||
|
||||
count, err := newInstanceQ(c.bun, (*models.Instance)(nil)).
|
||||
Count(ctx)
|
||||
count, err := newInstanceQ(c.bun, (*models.Instance)(nil)).Count(ctx)
|
||||
if err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -31,6 +31,10 @@ func (c *Client) CountInstances(ctx context.Context) (int64, db.Error) {
|
||||
func (c *Client) CreateInstance(ctx context.Context, instance *models.Instance) db.Error {
|
||||
metric := c.metrics.NewDBQuery("CreateInstance")
|
||||
|
||||
now := time.Now()
|
||||
instance.CreatedAt = now
|
||||
instance.UpdatedAt = now
|
||||
|
||||
if err := doCreate(ctx, c.bun, instance); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
@ -260,6 +264,8 @@ func (c *Client) ReadInstancesWhereFollowing(ctx context.Context) ([]*models.Ins
|
||||
func (c *Client) UpdateInstance(ctx context.Context, instance *models.Instance) db.Error {
|
||||
metric := c.metrics.NewDBQuery("UpdateInstance")
|
||||
|
||||
instance.UpdatedAt = time.Now()
|
||||
|
||||
if err := doUpdate(ctx, c.bun, instance); err != nil {
|
||||
go metric.Done(true)
|
||||
|
||||
|
||||
57
internal/db/bun/migrations/20220730130134_new.go
Normal file
57
internal/db/bun/migrations/20220730130134_new.go
Normal file
@ -0,0 +1,57 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect/sqltype"
|
||||
)
|
||||
|
||||
type columnList struct {
|
||||
Model interface{}
|
||||
Name string
|
||||
Type string
|
||||
}
|
||||
|
||||
func init() {
|
||||
l := logger.WithField("migration", "20220730130134")
|
||||
|
||||
up := func(ctx context.Context, db *bun.DB) error {
|
||||
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
addColumns := []columnList{
|
||||
{
|
||||
Model: (*models.Config)(nil),
|
||||
Name: "account_id",
|
||||
Type: sqltype.BigInt,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range addColumns {
|
||||
query := tx.NewAddColumn().
|
||||
Model(c.Model).
|
||||
ColumnExpr(fmt.Sprintf("%s %s", c.Name, c.Type))
|
||||
|
||||
l.Infof("creating column %s in table `%s`", c.Name, query.GetTableName())
|
||||
|
||||
_, err := query.Exec(ctx)
|
||||
if err != nil {
|
||||
l.Errorf("can't create column %s in table `%s`: %s", c.Name, query.GetTableName(), err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
down := func(ctx context.Context, db *bun.DB) error {
|
||||
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := Migrations.Register(up, down); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@ -70,8 +70,12 @@ type DB interface {
|
||||
DeleteConfigTX(ctx context.Context, txID TxID, configs ...*models.Config) (err Error)
|
||||
// ReadConfig returns one config.
|
||||
ReadConfig(ctx context.Context, id int64) (config *models.Config, err Error)
|
||||
// ReadConfigsForKeys returns configs for matching keys.
|
||||
ReadConfigsForKeys(ctx context.Context, keys ...models.ConfigKey) (configs []*models.Config, err Error)
|
||||
// ReadConfigsForAccountForKeys returns configs for matching keys.
|
||||
ReadConfigsForAccountForKeys(ctx context.Context, accountID int64, keys ...models.ConfigKey) (configs []*models.Config, err Error)
|
||||
// ReadConfigsForKeysSystem returns configs for matching keys.
|
||||
ReadConfigsForKeysSystem(ctx context.Context, keys ...models.ConfigKey) (configs []*models.Config, err Error)
|
||||
// ReadConfigsForKeysUser returns configs for matching keys.
|
||||
ReadConfigsForKeysUser(ctx context.Context, keys ...models.ConfigKey) (configs []*models.Config, err Error)
|
||||
// UpdateConfig updates configs.
|
||||
UpdateConfig(ctx context.Context, configs ...*models.Config) (err Error)
|
||||
// UpdateConfigTX updates configs using a transaction.
|
||||
|
||||
@ -10,4 +10,7 @@ type AdminHome struct {
|
||||
Common
|
||||
|
||||
FormHomeBody libtemplate.FormTextarea
|
||||
|
||||
FormNotificationTelegramEnabled libtemplate.FormInput
|
||||
FormNotificationTelegramToken libtemplate.FormInput
|
||||
}
|
||||
|
||||
@ -17,11 +17,10 @@ type Common struct {
|
||||
FooterScripts []libtemplate.Script
|
||||
FooterExtraScript []string
|
||||
HeadLinks []libtemplate.HeadLink
|
||||
LogoSrcDark string
|
||||
LogoSrcLight string
|
||||
NavBar Navbar
|
||||
NavBarDark bool
|
||||
PageTitle string
|
||||
|
||||
NavBar Navbar
|
||||
NavBarDark bool
|
||||
PageTitle string
|
||||
}
|
||||
|
||||
// AddHeadLink adds a headder link to the template.
|
||||
@ -58,12 +57,6 @@ func (t *Common) SetLocalizer(l *language.Localizer) {
|
||||
t.Localizer = l
|
||||
}
|
||||
|
||||
// SetLogoSrc sets the src for the logo image.
|
||||
func (t *Common) SetLogoSrc(dark, light string) {
|
||||
t.LogoSrcDark = dark
|
||||
t.LogoSrcLight = light
|
||||
}
|
||||
|
||||
// SetNavbar sets the top level navbar used by the template.
|
||||
func (t *Common) SetNavbar(nodes Navbar) {
|
||||
t.NavBar = nodes
|
||||
|
||||
@ -2,14 +2,20 @@ package template
|
||||
|
||||
import "github.com/feditools/relay/internal/path"
|
||||
|
||||
func GenActor(domain string) interface{} {
|
||||
func genActor(domain string) interface{} {
|
||||
return func() string {
|
||||
return path.GenActor(domain).String()
|
||||
}
|
||||
}
|
||||
|
||||
func GenInbox(domain string) interface{} {
|
||||
func genInbox(domain string) interface{} {
|
||||
return func() string {
|
||||
return path.GenInbox(domain).String()
|
||||
}
|
||||
}
|
||||
|
||||
func genString(s string) interface{} {
|
||||
return func() string {
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
14
internal/http/template/settings_home.go
Normal file
14
internal/http/template/settings_home.go
Normal file
@ -0,0 +1,14 @@
|
||||
package template
|
||||
|
||||
import libtemplate "github.com/feditools/go-lib/template"
|
||||
|
||||
// SettingsHomeName is the name of the home template.
|
||||
const SettingsHomeName = "settings_home"
|
||||
|
||||
// SettingsHome contains the variables for the home template.
|
||||
type SettingsHome struct {
|
||||
Common
|
||||
|
||||
FormNotificationTelegramEnabled libtemplate.FormInput
|
||||
FormNotificationTelegramChatID libtemplate.FormInput
|
||||
}
|
||||
@ -23,7 +23,6 @@ type InitTemplate interface {
|
||||
SetAccount(account *models.Account)
|
||||
SetLanguage(l string)
|
||||
SetLocalizer(l *language.Localizer)
|
||||
SetLogoSrc(dark, light string)
|
||||
SetNavbar(nodes Navbar)
|
||||
SetNavbarDark(dark bool)
|
||||
}
|
||||
@ -32,7 +31,11 @@ type InitTemplate interface {
|
||||
func New(tokz *token.Tokenizer) (*template.Template, error) {
|
||||
domain := viper.GetString(config.Keys.ServerExternalHostname)
|
||||
funMap := template.FuncMap{
|
||||
"applicationName": genString(viper.GetString(config.Keys.ApplicationName)),
|
||||
"applicationWebsite": genString(viper.GetString(config.Keys.ApplicationWebsite)),
|
||||
"token": tokz.GetToken,
|
||||
"logoSrcDark": genString(viper.GetString(config.Keys.WebappLogoSrcDark)),
|
||||
"logoSrcLight": genString(viper.GetString(config.Keys.WebappLogoSrcLight)),
|
||||
"pathAppAdminBlockExport": path.GenAppAdminBlockExport,
|
||||
"pathAppAdminBlockExportCSV": path.GenAppAdminBlockExportCSV,
|
||||
"pathAppAdminBlockExportJSON": path.GenAppAdminBlockExportJSON,
|
||||
@ -40,8 +43,10 @@ func New(tokz *token.Tokenizer) (*template.Template, error) {
|
||||
"pathAppHome": path.GenAppHomePath,
|
||||
"pathAppLogin": path.GenAppLoginPath,
|
||||
"pathAppLogout": path.GenAppLogoutPath,
|
||||
"urlActor": GenActor(domain),
|
||||
"urlInbox": GenInbox(domain),
|
||||
"pathAppSettingsHome": path.GenAppSettingsHomePath,
|
||||
"softwareVersion": genString(viper.GetString(config.Keys.SoftwareVersion)),
|
||||
"urlActor": genActor(domain),
|
||||
"urlInbox": genInbox(domain),
|
||||
}
|
||||
|
||||
tpl, err := libtemplate.New(funMap)
|
||||
|
||||
@ -16,7 +16,7 @@ func makeAdminNavbar(r *nethttp.Request) template.Navbar {
|
||||
{
|
||||
Text: l.TextConfig(1).String(),
|
||||
MatchStr: path.ReAppAdminHome,
|
||||
FAIcon: "server",
|
||||
FAIcon: "gear",
|
||||
URL: path.AppAdminHome,
|
||||
},
|
||||
{
|
||||
|
||||
@ -102,34 +102,40 @@ func (m *Module) displayAdminBlockList(w http.ResponseWriter, r *http.Request, c
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormDomain,
|
||||
Placeholder: "example.com",
|
||||
Label: localizer.TextDomain(1),
|
||||
LabelClass: "form-label",
|
||||
Value: config.FormAddDomainValue,
|
||||
Disabled: false,
|
||||
Required: true,
|
||||
Validation: config.FormAddDomainValidation,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextDomain(1),
|
||||
Class: "form-label",
|
||||
},
|
||||
Value: config.FormAddDomainValue,
|
||||
Disabled: false,
|
||||
Required: true,
|
||||
Validation: config.FormAddDomainValidation,
|
||||
}
|
||||
tmplVars.FormAddObfuscatedDomain = libtemplate.FormInput{
|
||||
ID: "formAddInputObfuscatedDomain",
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormObfuscatedDomain,
|
||||
Placeholder: "e*****e.com",
|
||||
Label: localizer.TextObfuscatedDomain(1),
|
||||
LabelClass: "form-label",
|
||||
Value: config.FormAddObfuscatedDomainValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormAddObfuscatedDomainValidation,
|
||||
}
|
||||
tmplVars.FormAddBlockSubdomains = libtemplate.FormInput{
|
||||
ID: "formAddInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: localizer.TextBlockSubdomain(2),
|
||||
LabelClass: "form-check-label",
|
||||
Checked: config.FormAddBlockSubdomainsValue,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextObfuscatedDomain(1),
|
||||
Class: "form-label",
|
||||
},
|
||||
Value: config.FormAddObfuscatedDomainValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormAddObfuscatedDomainValidation,
|
||||
}
|
||||
tmplVars.FormAddBlockSubdomains = libtemplate.FormInput{
|
||||
ID: "formAddInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextBlockSubdomain(2),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Checked: config.FormAddBlockSubdomainsValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
}
|
||||
|
||||
// create edit form inputs
|
||||
@ -150,34 +156,40 @@ func (m *Module) displayAdminBlockList(w http.ResponseWriter, r *http.Request, c
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormDomain,
|
||||
Placeholder: "example.com",
|
||||
Label: localizer.TextDomain(1),
|
||||
LabelClass: "form-label",
|
||||
Value: config.FormAddDomainValue,
|
||||
Disabled: false,
|
||||
Required: true,
|
||||
Validation: config.FormAddDomainValidation,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextDomain(2),
|
||||
Class: "form-label",
|
||||
},
|
||||
Value: config.FormAddDomainValue,
|
||||
Disabled: false,
|
||||
Required: true,
|
||||
Validation: config.FormAddDomainValidation,
|
||||
}
|
||||
tmplVars.FormEditObfuscatedDomain = libtemplate.FormInput{
|
||||
ID: "formEditInputObfuscatedDomain",
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormObfuscatedDomain,
|
||||
Placeholder: "e*****e.com",
|
||||
Label: localizer.TextObfuscatedDomain(1),
|
||||
LabelClass: "form-label",
|
||||
Value: config.FormAddObfuscatedDomainValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormAddObfuscatedDomainValidation,
|
||||
}
|
||||
tmplVars.FormEditBlockSubdomains = libtemplate.FormInput{
|
||||
ID: "formEditInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: localizer.TextBlockSubdomain(2),
|
||||
LabelClass: "form-check-label",
|
||||
Checked: config.FormAddBlockSubdomainsValue,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextObfuscatedDomain(1),
|
||||
Class: "form-label",
|
||||
},
|
||||
Value: config.FormAddObfuscatedDomainValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormAddObfuscatedDomainValidation,
|
||||
}
|
||||
tmplVars.FormEditBlockSubdomains = libtemplate.FormInput{
|
||||
ID: "formEditInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextBlockSubdomain(2),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Checked: config.FormAddBlockSubdomainsValue,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
}
|
||||
|
||||
// create delete form inputs
|
||||
@ -202,21 +214,25 @@ func (m *Module) displayAdminBlockList(w http.ResponseWriter, r *http.Request, c
|
||||
Value: ActionImport,
|
||||
}
|
||||
tmplVars.FormImportFile = libtemplate.FormInput{
|
||||
ID: "formImportInputFile",
|
||||
Type: libtemplate.FormInputTypeFile,
|
||||
Name: FormFile,
|
||||
Label: localizer.TextObfuscatedDomain(1),
|
||||
LabelClass: "form-label",
|
||||
ID: "formImportInputFile",
|
||||
Type: libtemplate.FormInputTypeFile,
|
||||
Name: FormFile,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextObfuscatedDomain(1),
|
||||
Class: "form-label",
|
||||
},
|
||||
}
|
||||
tmplVars.FormImportBlockSubdomains = libtemplate.FormInput{
|
||||
ID: "formImportInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: localizer.TextBlockSubdomain(2),
|
||||
LabelClass: "form-check-label",
|
||||
Checked: true,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
ID: "formImportInputSubdomain",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormSubdomain,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextBlockSubdomain(2),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Checked: true,
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
}
|
||||
|
||||
// add javascript
|
||||
@ -268,8 +284,5 @@ func (m *Module) displayAdminBlockList(w http.ResponseWriter, r *http.Request, c
|
||||
}
|
||||
tmplVars.Pagination = libtemplate.MakePagination(pageConf)
|
||||
|
||||
err = m.executeTemplate(w, template.AdminBlockName, tmplVars)
|
||||
if err != nil {
|
||||
l.Errorf("could not render '%s' template: %s", template.AdminBlockName, err.Error())
|
||||
}
|
||||
m.executeTemplate(w, template.AdminBlockName, tmplVars)
|
||||
}
|
||||
|
||||
@ -81,7 +81,7 @@ func (m *Module) doAddBlock(w http.ResponseWriter, r *http.Request) {
|
||||
DisplayModal: "addModal",
|
||||
|
||||
FormAddError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: localizer.TextBlockExists(domain).String(),
|
||||
},
|
||||
|
||||
@ -113,7 +113,7 @@ func (m *Module) doAddBlock(w http.ResponseWriter, r *http.Request) {
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "Block created.",
|
||||
},
|
||||
},
|
||||
|
||||
@ -56,7 +56,7 @@ func (m *Module) doDeleteBlock(w http.ResponseWriter, r *http.Request) {
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "Block deleted.",
|
||||
},
|
||||
},
|
||||
|
||||
@ -97,7 +97,7 @@ func (m *Module) doEditBlock(w http.ResponseWriter, r *http.Request) {
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "Block updated.",
|
||||
},
|
||||
},
|
||||
|
||||
@ -48,7 +48,7 @@ func (m *Module) doImportBlocks(w http.ResponseWriter, r *http.Request) {
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Unknown file type %s.", filename[len(filename)-1]),
|
||||
},
|
||||
})
|
||||
@ -70,7 +70,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: msg,
|
||||
},
|
||||
})
|
||||
@ -87,7 +87,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Block object %d missing domain", i),
|
||||
},
|
||||
})
|
||||
@ -103,7 +103,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Error checking for block %s: %s", fileBlock.Domain, err.Error()),
|
||||
},
|
||||
})
|
||||
@ -122,7 +122,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Block object %d has invalid domain obfuscation", i),
|
||||
},
|
||||
})
|
||||
@ -143,7 +143,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: localizer.TextErrorDatabase().String(),
|
||||
},
|
||||
})
|
||||
@ -156,7 +156,7 @@ func (m *Module) doImportBlocksAppJSON(w http.ResponseWriter, r *http.Request, f
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: fmt.Sprintf("Imported %d blocks", len(blocksToAdd)),
|
||||
},
|
||||
},
|
||||
@ -179,7 +179,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: msg,
|
||||
},
|
||||
})
|
||||
@ -209,7 +209,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: msg,
|
||||
},
|
||||
})
|
||||
@ -235,7 +235,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Error checking for block %s: %s", domain, err.Error()),
|
||||
},
|
||||
})
|
||||
@ -261,7 +261,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Block line %d has invalid domain obfuscation", i),
|
||||
},
|
||||
})
|
||||
@ -295,7 +295,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: localizer.TextErrorDatabase().String(),
|
||||
},
|
||||
})
|
||||
@ -308,7 +308,7 @@ func (m *Module) doImportBlocksTextCSV(w http.ResponseWriter, r *http.Request, f
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Imported %d blocks", len(blocksToAdd)),
|
||||
},
|
||||
},
|
||||
@ -347,7 +347,7 @@ func (m *Module) doImportBlocksTextPlain(w http.ResponseWriter, r *http.Request,
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: fmt.Sprintf("Error checking for block %s: %s", domainString, err.Error()),
|
||||
},
|
||||
})
|
||||
@ -376,7 +376,7 @@ func (m *Module) doImportBlocksTextPlain(w http.ResponseWriter, r *http.Request,
|
||||
DisplayModal: "importModal",
|
||||
|
||||
FormInputError: &libtemplate.Alert{
|
||||
Level: "danger",
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: localizer.TextErrorDatabase().String(),
|
||||
},
|
||||
})
|
||||
@ -389,7 +389,7 @@ func (m *Module) doImportBlocksTextPlain(w http.ResponseWriter, r *http.Request,
|
||||
m.displayAdminBlockList(w, r, displayAdminBlockListConfig{
|
||||
Alerts: &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: fmt.Sprintf("Imported %d blocks", len(blocksToAdd)),
|
||||
},
|
||||
},
|
||||
|
||||
@ -25,22 +25,9 @@ func (m *Module) AdminHomeGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
displayConfig := displayAdminHomeConfig{}
|
||||
|
||||
// config home body
|
||||
if (*configMap)[models.ConfigHomeBody] != nil {
|
||||
homeBody, err := (*configMap)[models.ConfigHomeBody].GetValue()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigHomeBody, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, "")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
displayConfig.FormHomeBodyValue = homeBody
|
||||
}
|
||||
|
||||
m.displayAdminHome(w, r, displayConfig)
|
||||
m.displayAdminHome(w, r, displayAdminHomeConfig{
|
||||
ConfigMap: configMap,
|
||||
})
|
||||
}
|
||||
|
||||
// AdminHomePostHandler handles the admin home page form.
|
||||
@ -50,6 +37,9 @@ func (m *Module) AdminHomePostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// get localizer
|
||||
localizer := r.Context().Value(ContextKeyLocalizer).(*language.Localizer) //nolint
|
||||
|
||||
// display page
|
||||
displayConfig := displayAdminHomeConfig{}
|
||||
|
||||
// get configMap
|
||||
configMap, err := m.adminHomeConfigMap(r.Context())
|
||||
if err != nil {
|
||||
@ -75,9 +65,57 @@ func (m *Module) AdminHomePostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
return
|
||||
}
|
||||
err = m.adminHomeFormTelegramEnabled(r, configMap, &changes)
|
||||
if err != nil {
|
||||
l.Errorf("form telegram enabled: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
err = m.adminHomeFormTelegramToken(r, configMap, &changes)
|
||||
if err != nil {
|
||||
l.Errorf("form telegram token: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
displayConfig.ConfigMap = configMap
|
||||
|
||||
// validate new config
|
||||
validationError := false
|
||||
|
||||
// enabling telegram requires token to be set
|
||||
if (*configMap)[models.ConfigTelegramEnabled] != nil {
|
||||
value, err := (*configMap)[models.ConfigTelegramEnabled].GetValueBool()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigTelegramEnabled, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
if value && (*configMap)[models.ConfigTelegramToken] == nil {
|
||||
displayConfig.FormNotificationTelegramTokenValidation = &libtemplate.FormValidation{
|
||||
Valid: false,
|
||||
Response: "Token must be set to enable Telegram notifications.",
|
||||
}
|
||||
|
||||
validationError = true
|
||||
}
|
||||
}
|
||||
|
||||
if validationError {
|
||||
displayConfig.Alerts = &[]libtemplate.Alert{
|
||||
{
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: "validation error, config not saved",
|
||||
},
|
||||
}
|
||||
m.displayAdminHome(w, r, displayConfig)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// update db
|
||||
var alerts *[]libtemplate.Alert
|
||||
if len(changes) > 0 {
|
||||
err = m.logic.ProcessConfigChanges(r.Context(), changes)
|
||||
if err != nil {
|
||||
@ -87,94 +125,23 @@ func (m *Module) AdminHomePostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
alerts = &[]libtemplate.Alert{
|
||||
displayConfig.Alerts = &[]libtemplate.Alert{
|
||||
{
|
||||
Level: "success",
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "config updated",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// display page
|
||||
displayConfig := displayAdminHomeConfig{
|
||||
Alerts: alerts,
|
||||
}
|
||||
|
||||
m.displayAdminHome(w, r, displayConfig)
|
||||
}
|
||||
|
||||
func (m *Module) adminHomeFormHomeBody(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "adminHomeFormHomeBody")
|
||||
|
||||
formValue := r.FormValue(FormHomeBody)
|
||||
switch {
|
||||
case (*cm)[models.ConfigHomeBody] == nil && formValue != "":
|
||||
l.Debugf("adding config %s", models.ConfigHomeBody)
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigHomeBody,
|
||||
}
|
||||
|
||||
err := config.SetValue(formValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigHomeBody] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigHomeBody] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigHomeBody]
|
||||
value, err := config.GetValue()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value == formValue {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s", models.ConfigHomeBody)
|
||||
|
||||
// set value
|
||||
err = config.SetValue(formValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigHomeBody] != nil && formValue == "":
|
||||
l.Debugf("deleting config %s", models.ConfigHomeBody)
|
||||
|
||||
// delete current config
|
||||
config := (*cm)[models.ConfigHomeBody]
|
||||
(*cm)[models.ConfigHomeBody] = nil
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeDelete,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type displayAdminHomeConfig struct {
|
||||
Alerts *[]libtemplate.Alert
|
||||
Alerts *[]libtemplate.Alert
|
||||
ConfigMap *models.ConfigMap
|
||||
|
||||
FormHomeBodyValue string
|
||||
FormHomeBodyValidation *libtemplate.FormValidation
|
||||
FormHomeBodyValidation *libtemplate.FormValidation
|
||||
FormNotificationTelegramTokenValidation *libtemplate.FormValidation
|
||||
}
|
||||
|
||||
func (m *Module) displayAdminHome(w http.ResponseWriter, r *http.Request, config displayAdminHomeConfig) {
|
||||
@ -191,16 +158,45 @@ func (m *Module) displayAdminHome(w http.ResponseWriter, r *http.Request, config
|
||||
},
|
||||
|
||||
FormHomeBody: libtemplate.FormTextarea{
|
||||
ID: "formHomeBody",
|
||||
Name: FormHomeBody,
|
||||
Rows: 8,
|
||||
Label: localizer.TextHomePageBody(),
|
||||
LabelClass: "form-label",
|
||||
ID: "formHomeBody",
|
||||
Name: FormHomeBody,
|
||||
Rows: 8,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextHomePageBody(),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Value: config.FormHomeBodyValue,
|
||||
Validation: config.FormHomeBodyValidation,
|
||||
},
|
||||
|
||||
FormNotificationTelegramEnabled: libtemplate.FormInput{
|
||||
ID: "formNotificationTelegramEnabled",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormTelegramEnabled,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextEnabled(),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
},
|
||||
FormNotificationTelegramToken: libtemplate.FormInput{
|
||||
ID: "formNotificationTelegramToken",
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormTelegramToken,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextToken(1),
|
||||
Class: "form-label",
|
||||
Badge: &libtemplate.Badge{
|
||||
Color: libtemplate.ColorSecondary,
|
||||
Text: "unset",
|
||||
},
|
||||
},
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormNotificationTelegramTokenValidation,
|
||||
},
|
||||
}
|
||||
err := m.initTemplateAdmin(w, r, tmplVars)
|
||||
if err != nil {
|
||||
@ -209,12 +205,43 @@ func (m *Module) displayAdminHome(w http.ResponseWriter, r *http.Request, config
|
||||
return
|
||||
}
|
||||
|
||||
err = m.executeTemplate(w, template.AdminHomeName, tmplVars)
|
||||
if err != nil {
|
||||
l.Errorf("could not render '%s' template: %s", template.AdminHomeName, err.Error())
|
||||
// config form values
|
||||
if (*config.ConfigMap)[models.ConfigHomeBody] != nil {
|
||||
value, err := (*config.ConfigMap)[models.ConfigHomeBody].GetValue()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigHomeBody, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, "")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
tmplVars.FormHomeBody.Value = value
|
||||
}
|
||||
if (*config.ConfigMap)[models.ConfigTelegramEnabled] != nil {
|
||||
value, err := (*config.ConfigMap)[models.ConfigTelegramEnabled].GetValueBool()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigTelegramEnabled, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
tmplVars.FormNotificationTelegramEnabled.Checked = value
|
||||
}
|
||||
if (*config.ConfigMap)[models.ConfigTelegramToken] != nil {
|
||||
tmplVars.FormNotificationTelegramToken.Label.Badge = &libtemplate.Badge{
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "set",
|
||||
}
|
||||
}
|
||||
|
||||
m.executeTemplate(w, template.AdminHomeName, tmplVars)
|
||||
}
|
||||
|
||||
func (m *Module) adminHomeConfigMap(ctx context.Context) (*models.ConfigMap, error) {
|
||||
return m.logic.GetConfigMap(ctx, []models.ConfigKey{models.ConfigHomeBody}...)
|
||||
return m.logic.GetConfigMap(ctx, []models.ConfigKey{
|
||||
models.ConfigHomeBody,
|
||||
models.ConfigTelegramEnabled,
|
||||
models.ConfigTelegramToken,
|
||||
}...)
|
||||
}
|
||||
|
||||
207
internal/http/webapp/admin_home_form.go
Normal file
207
internal/http/webapp/admin_home_form.go
Normal file
@ -0,0 +1,207 @@
|
||||
package webapp
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func (m *Module) adminHomeFormHomeBody(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "adminHomeFormHomeBody")
|
||||
|
||||
formValue := r.FormValue(FormHomeBody)
|
||||
switch {
|
||||
case (*cm)[models.ConfigHomeBody] == nil && formValue != "":
|
||||
l.Debugf("adding config %s", models.ConfigHomeBody)
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigHomeBody,
|
||||
}
|
||||
|
||||
err := config.SetValue(formValue, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigHomeBody] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigHomeBody] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigHomeBody]
|
||||
value, err := config.GetValue()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value == formValue {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s", models.ConfigHomeBody)
|
||||
|
||||
// set value
|
||||
err = config.SetValue(formValue, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigHomeBody] != nil && formValue == "":
|
||||
l.Debugf("deleting config %s", models.ConfigHomeBody)
|
||||
|
||||
// delete current config
|
||||
config := (*cm)[models.ConfigHomeBody]
|
||||
(*cm)[models.ConfigHomeBody] = nil
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeDelete,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) adminHomeFormTelegramEnabled(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "adminHomeFormTelegramEnabled")
|
||||
|
||||
formValue := r.FormValue(FormTelegramEnabled)
|
||||
switch {
|
||||
case (*cm)[models.ConfigTelegramEnabled] == nil && formValue != "":
|
||||
l.Debugf("adding config %s (true)", models.ConfigTelegramEnabled)
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigTelegramEnabled,
|
||||
}
|
||||
err := config.SetValueBool(true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigTelegramEnabled] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigTelegramEnabled] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigTelegramEnabled]
|
||||
value, err := config.GetValueBool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s (true)", models.ConfigTelegramEnabled)
|
||||
|
||||
// set value
|
||||
err = config.SetValueBool(true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigTelegramEnabled] != nil && formValue == "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigTelegramEnabled]
|
||||
value, err := config.GetValueBool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !value {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s (false)", models.ConfigTelegramEnabled)
|
||||
|
||||
// set value
|
||||
err = config.SetValueBool(false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) adminHomeFormTelegramToken(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "adminHomeFormTelegramToken")
|
||||
|
||||
formValue := r.FormValue(FormTelegramToken)
|
||||
switch {
|
||||
case (*cm)[models.ConfigTelegramToken] == nil && formValue != "":
|
||||
l.Debugf("adding config %s", models.ConfigTelegramToken)
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigTelegramToken,
|
||||
}
|
||||
|
||||
err := config.SetValue(formValue, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigTelegramToken] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigTelegramToken] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigTelegramToken]
|
||||
value, err := config.GetValue()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value == formValue {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s", models.ConfigTelegramToken)
|
||||
|
||||
// set value
|
||||
err = config.SetValue(formValue, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -72,8 +72,5 @@ func (m *Module) displayAdminInstance(w http.ResponseWriter, r *http.Request, co
|
||||
}
|
||||
tmplVars.Pagination = libtemplate.MakePagination(pageConf)
|
||||
|
||||
err = m.executeTemplate(w, template.AdminInstanceName, tmplVars)
|
||||
if err != nil {
|
||||
l.Errorf("could not render '%s' template: %s", template.AdminInstanceName, err.Error())
|
||||
}
|
||||
m.executeTemplate(w, template.AdminInstanceName, tmplVars)
|
||||
}
|
||||
|
||||
@ -7,9 +7,6 @@ const (
|
||||
// COAnonymous is an anonymous cross origin.
|
||||
COAnonymous = "anonymous"
|
||||
|
||||
// ErrorResponseDBError is returned to the user when there is a database error
|
||||
//ErrorResponseDBError = "database error"
|
||||
|
||||
// ActionAdd is the value for an add action.
|
||||
ActionAdd = "add"
|
||||
// ActionDelete is the value for a delete action.
|
||||
@ -38,6 +35,17 @@ const (
|
||||
FormObfuscatedDomain = "obfuscated-domain"
|
||||
// FormSubdomain is the key for a subdomain form field.
|
||||
FormSubdomain = "subdomain"
|
||||
// FormTelegramChatID is the key for a chat ID form field.
|
||||
FormTelegramChatID = "telegram-chat-id"
|
||||
// FormTelegramChatIDType is the key for a chat ID type form field.
|
||||
FormTelegramChatIDType = "telegram-chat-id-type"
|
||||
// FormTelegramEnabled is the key for a telegram enabled form field.
|
||||
FormTelegramEnabled = "telegram-enabled"
|
||||
// FormTelegramToken is the key for a telegram token form field.
|
||||
FormTelegramToken = "telegram-token"
|
||||
// FormToken is the key for a token form field.
|
||||
FormToken = "token"
|
||||
|
||||
FormValueUsername = "username"
|
||||
FormValueChatID = "chat-id"
|
||||
)
|
||||
|
||||
@ -53,10 +53,7 @@ func (m *Module) returnErrorPage(w http.ResponseWriter, r *http.Request, code in
|
||||
}
|
||||
|
||||
w.WriteHeader(code)
|
||||
err = m.executeTemplate(w, "error", tmplVars)
|
||||
if err != nil {
|
||||
logger.Errorf("could not render error template: %s", err.Error())
|
||||
}
|
||||
m.executeTemplate(w, "error", tmplVars)
|
||||
}
|
||||
|
||||
func (m *Module) methodNotAllowedHandler() http.Handler {
|
||||
|
||||
@ -75,10 +75,7 @@ func (m *Module) HomeGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
tmplVars.Blocks = blocks
|
||||
|
||||
err = m.executeTemplate(w, template.HomeName, tmplVars)
|
||||
if err != nil {
|
||||
l.Errorf("could not render '%s' template: %s", template.HomeName, err.Error())
|
||||
}
|
||||
m.executeTemplate(w, template.HomeName, tmplVars)
|
||||
}
|
||||
|
||||
// ForwardToHomeHandler serves a home forwarder.
|
||||
|
||||
@ -108,8 +108,5 @@ func (m *Module) displayLoginPage(w nethttp.ResponseWriter, r *nethttp.Request,
|
||||
tmplVars.FormError = formError
|
||||
tmplVars.FormAccount = account
|
||||
|
||||
err = m.executeTemplate(w, template.LoginName, tmplVars)
|
||||
if err != nil {
|
||||
l.Errorf("could not render %s template: %s", template.LoginName, err.Error())
|
||||
}
|
||||
m.executeTemplate(w, template.LoginName, tmplVars)
|
||||
}
|
||||
|
||||
@ -69,6 +69,18 @@ func (m *Module) Middleware(next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// MiddlewareRequireAuth will redirect a user to login page if user not in context.
|
||||
func (m *Module) MiddlewareRequireAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, shouldReturn := m.authRequireLoggedIn(w, r)
|
||||
if shouldReturn {
|
||||
return
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// MiddlewareRequireAdmin will redirect a user to login page if user not in context and will return unauthorized for
|
||||
// a non admin user.
|
||||
func (m *Module) MiddlewareRequireAdmin(next http.Handler) http.Handler {
|
||||
|
||||
@ -22,6 +22,14 @@ func (m *Module) Route(s *http.Server) error {
|
||||
webapp.HandleFunc(path.AppSubLogin, m.LoginPostHandler).Methods(nethttp.MethodPost)
|
||||
webapp.HandleFunc(path.AppSubLogout, m.LogoutGetHandler).Methods(nethttp.MethodGet)
|
||||
|
||||
settings := webapp.PathPrefix(path.AppSubSettings).Subrouter()
|
||||
settings.Use(m.MiddlewareRequireAuth)
|
||||
settings.NotFoundHandler = m.notFoundHandler()
|
||||
settings.MethodNotAllowedHandler = m.methodNotAllowedHandler()
|
||||
|
||||
settings.HandleFunc(path.AppSettingsSubHome, m.SettingsHomeGetHandler).Methods(nethttp.MethodGet)
|
||||
settings.HandleFunc(path.AppSettingsSubHome, m.SettingsHomePostHandler).Methods(nethttp.MethodPost)
|
||||
|
||||
admin := webapp.PathPrefix(path.AppAdmin).Subrouter()
|
||||
admin.Use(m.MiddlewareRequireAdmin)
|
||||
admin.NotFoundHandler = m.notFoundHandler()
|
||||
|
||||
203
internal/http/webapp/settings_home.go
Normal file
203
internal/http/webapp/settings_home.go
Normal file
@ -0,0 +1,203 @@
|
||||
package webapp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/feditools/go-lib/language"
|
||||
libtemplate "github.com/feditools/go-lib/template"
|
||||
"github.com/feditools/relay/internal/http/template"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// SettingsHomeGetHandler serves the home page.
|
||||
func (m *Module) SettingsHomeGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
l := logger.WithField("func", "SettingsHomeGetHandler")
|
||||
|
||||
// get localizer
|
||||
localizer := r.Context().Value(ContextKeyLocalizer).(*language.Localizer) //nolint
|
||||
|
||||
// account
|
||||
account := r.Context().Value(ContextKeyAccount).(*models.Account) //nolint
|
||||
|
||||
// get configMap
|
||||
configMap, err := m.settingsHomeConfigMap(r.Context(), account.ID)
|
||||
if err != nil {
|
||||
l.Errorf("get config: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, localizer.TextErrorDatabase().String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
m.displaySettingsHome(w, r, displaySettingsHomeConfig{
|
||||
ConfigMap: configMap,
|
||||
})
|
||||
}
|
||||
|
||||
// SettingsHomePostHandler serves the home page.
|
||||
func (m *Module) SettingsHomePostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
l := logger.WithField("func", "SettingsHomePostHandler")
|
||||
|
||||
// get localizer
|
||||
localizer := r.Context().Value(ContextKeyLocalizer).(*language.Localizer) //nolint
|
||||
|
||||
// account
|
||||
account := r.Context().Value(ContextKeyAccount).(*models.Account) //nolint
|
||||
|
||||
// display page
|
||||
displayConfig := displaySettingsHomeConfig{}
|
||||
|
||||
// get configMap
|
||||
configMap, err := m.settingsHomeConfigMap(r.Context(), account.ID)
|
||||
if err != nil {
|
||||
l.Errorf("get config: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, localizer.TextErrorDatabase().String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// gather changes
|
||||
var changes []*models.ConfigChange
|
||||
err = m.settingsHomeFormTelegramEnabled(r, configMap, &changes)
|
||||
if err != nil {
|
||||
l.Errorf("form home body: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
err = m.settingsHomeFormTelegramChatID(r, configMap, &changes)
|
||||
if err != nil {
|
||||
l.Errorf("form telegram enabled: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
displayConfig.ConfigMap = configMap
|
||||
|
||||
// validate form
|
||||
valid, err := settingsHomeFormValidate(configMap, &displayConfig)
|
||||
if err != nil {
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
if !valid {
|
||||
l.Tracef("form invalid, returning error")
|
||||
displayConfig.Alerts = &[]libtemplate.Alert{
|
||||
{
|
||||
Color: libtemplate.ColorDanger,
|
||||
Text: "validation error, settings not saved",
|
||||
},
|
||||
}
|
||||
m.displaySettingsHome(w, r, displayConfig)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// update db
|
||||
if len(changes) > 0 {
|
||||
err = m.logic.ProcessConfigChanges(r.Context(), changes)
|
||||
if err != nil {
|
||||
l.Errorf("db change configs: %s", err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, localizer.TextErrorDatabase().String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
displayConfig.Alerts = &[]libtemplate.Alert{
|
||||
{
|
||||
Color: libtemplate.ColorSuccess,
|
||||
Text: "config updated",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
m.displaySettingsHome(w, r, displayConfig)
|
||||
}
|
||||
|
||||
type displaySettingsHomeConfig struct {
|
||||
Alerts *[]libtemplate.Alert
|
||||
ConfigMap *models.ConfigMap
|
||||
|
||||
FormHomeBodyValidation *libtemplate.FormValidation
|
||||
FormNotificationTelegramChatIDValidation *libtemplate.FormValidation
|
||||
}
|
||||
|
||||
func (m *Module) displaySettingsHome(w http.ResponseWriter, r *http.Request, config displaySettingsHomeConfig) {
|
||||
l := logger.WithField("func", "displaySettingsHome")
|
||||
|
||||
// get localizer
|
||||
localizer := r.Context().Value(ContextKeyLocalizer).(*language.Localizer) //nolint
|
||||
|
||||
// Init template variables
|
||||
tmplVars := &template.SettingsHome{
|
||||
Common: template.Common{
|
||||
Alerts: config.Alerts,
|
||||
PageTitle: localizer.TextRelay(1).String(),
|
||||
},
|
||||
|
||||
FormNotificationTelegramEnabled: libtemplate.FormInput{
|
||||
ID: "formNotificationTelegramEnabled",
|
||||
Type: libtemplate.FormInputTypeCheckbox,
|
||||
Name: FormTelegramEnabled,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextEnabled(),
|
||||
Class: "form-check-label",
|
||||
},
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
},
|
||||
FormNotificationTelegramChatID: libtemplate.FormInput{
|
||||
ID: "formNotificationTelegramChatID",
|
||||
Type: libtemplate.FormInputTypeText,
|
||||
Name: FormTelegramChatID,
|
||||
Label: &libtemplate.FormLabel{
|
||||
Text: localizer.TextChatIDOrUsername(),
|
||||
Class: "form-label",
|
||||
},
|
||||
Disabled: false,
|
||||
Required: false,
|
||||
Validation: config.FormNotificationTelegramChatIDValidation,
|
||||
},
|
||||
}
|
||||
err := m.initTemplatePublic(w, r, tmplVars)
|
||||
if err != nil {
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// config form values
|
||||
if (*config.ConfigMap)[models.ConfigUserTelegramEnabled] != nil {
|
||||
value, err := (*config.ConfigMap)[models.ConfigUserTelegramEnabled].GetValueBool()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigUserTelegramEnabled, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, "")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
tmplVars.FormNotificationTelegramEnabled.Checked = value
|
||||
}
|
||||
if (*config.ConfigMap)[models.ConfigUserTelegramChatID] != nil {
|
||||
value, err := (*config.ConfigMap)[models.ConfigUserTelegramChatID].GetValue()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigUserTelegramChatID, err.Error())
|
||||
m.returnErrorPage(w, r, http.StatusInternalServerError, "")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
tmplVars.FormNotificationTelegramChatID.Value = value
|
||||
}
|
||||
|
||||
m.executeTemplate(w, template.SettingsHomeName, tmplVars)
|
||||
}
|
||||
|
||||
func (m *Module) settingsHomeConfigMap(ctx context.Context, accountID int64) (*models.ConfigMap, error) {
|
||||
return m.logic.GetConfigMapForAccount(
|
||||
ctx,
|
||||
accountID,
|
||||
models.ConfigUserTelegramEnabled,
|
||||
models.ConfigUserTelegramChatID,
|
||||
)
|
||||
}
|
||||
203
internal/http/webapp/settings_home_form.go
Normal file
203
internal/http/webapp/settings_home_form.go
Normal file
@ -0,0 +1,203 @@
|
||||
package webapp
|
||||
|
||||
import (
|
||||
libtemplate "github.com/feditools/go-lib/template"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func (m *Module) settingsHomeFormTelegramEnabled(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "settingsHomeFormTelegramEnabled")
|
||||
|
||||
formValue := r.FormValue(FormTelegramEnabled)
|
||||
switch {
|
||||
case (*cm)[models.ConfigUserTelegramEnabled] == nil && formValue != "":
|
||||
l.Debugf("adding config %s (true)", models.ConfigUserTelegramEnabled)
|
||||
|
||||
// account
|
||||
account := r.Context().Value(ContextKeyAccount).(*models.Account) //nolint
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigUserTelegramEnabled,
|
||||
AccountID: account.ID,
|
||||
Account: account,
|
||||
}
|
||||
err := config.SetValueBool(true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigUserTelegramEnabled] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigUserTelegramEnabled] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigUserTelegramEnabled]
|
||||
value, err := config.GetValueBool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s (true)", models.ConfigUserTelegramEnabled)
|
||||
|
||||
// set value
|
||||
err = config.SetValueBool(true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigUserTelegramEnabled] != nil && formValue == "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigUserTelegramEnabled]
|
||||
value, err := config.GetValueBool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !value {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s (false)", models.ConfigUserTelegramEnabled)
|
||||
|
||||
// set value
|
||||
err = config.SetValueBool(false, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) settingsHomeFormTelegramChatID(r *http.Request, cm *models.ConfigMap, changes *[]*models.ConfigChange) error {
|
||||
l := logger.WithField("func", "settingsHomeFormTelegramChatID")
|
||||
|
||||
formValue := r.FormValue(FormTelegramChatID)
|
||||
switch {
|
||||
case (*cm)[models.ConfigUserTelegramChatID] == nil && formValue != "":
|
||||
l.Debugf("adding config %s", models.ConfigUserTelegramChatID)
|
||||
|
||||
// account
|
||||
account := r.Context().Value(ContextKeyAccount).(*models.Account) //nolint
|
||||
|
||||
// create new config
|
||||
config := models.Config{
|
||||
Key: models.ConfigUserTelegramChatID,
|
||||
AccountID: account.ID,
|
||||
Account: account,
|
||||
}
|
||||
err := config.SetValue(formValue, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*cm)[models.ConfigUserTelegramChatID] = &config
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeAdd,
|
||||
Config: &config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigUserTelegramChatID] != nil && formValue != "":
|
||||
// update current config
|
||||
config := (*cm)[models.ConfigUserTelegramChatID]
|
||||
value, err := config.GetValue()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value == formValue {
|
||||
// nothing to do
|
||||
return nil
|
||||
}
|
||||
|
||||
l.Debugf("updatng config %s", models.ConfigUserTelegramChatID)
|
||||
|
||||
// set value
|
||||
err = config.SetValue(formValue, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeUpdate,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
case (*cm)[models.ConfigUserTelegramChatID] != nil && formValue == "":
|
||||
l.Debugf("deleting config %s", models.ConfigUserTelegramChatID)
|
||||
|
||||
// delete current config
|
||||
config := (*cm)[models.ConfigUserTelegramChatID]
|
||||
(*cm)[models.ConfigUserTelegramChatID] = nil
|
||||
|
||||
change := models.ConfigChange{
|
||||
Change: models.ChangeDelete,
|
||||
Config: config,
|
||||
}
|
||||
*changes = append(*changes, &change)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func settingsHomeFormValidate(cm *models.ConfigMap, displayConfig *displaySettingsHomeConfig) (bool, error) {
|
||||
l := logger.WithField("func", "settingsHomeFormValidate")
|
||||
|
||||
valid := true
|
||||
|
||||
// enabling telegram requires chat id to be set
|
||||
if (*cm)[models.ConfigUserTelegramEnabled] != nil {
|
||||
l.Tracef("telegram enabled")
|
||||
enabled, err := (*cm)[models.ConfigUserTelegramEnabled].GetValueBool()
|
||||
if err != nil {
|
||||
l.Errorf("reading config %s: %s", models.ConfigUserTelegramEnabled, err.Error())
|
||||
|
||||
return false, err
|
||||
}
|
||||
if enabled && (*cm)[models.ConfigUserTelegramChatID] == nil {
|
||||
l.Tracef("telegram chat id missing")
|
||||
displayConfig.FormNotificationTelegramChatIDValidation = &libtemplate.FormValidation{
|
||||
Valid: false,
|
||||
Response: "Chat ID must be set to enable Telegram notifications.",
|
||||
}
|
||||
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
|
||||
// chat id must be integer
|
||||
if (*cm)[models.ConfigUserTelegramChatID] != nil {
|
||||
_, err := (*cm)[models.ConfigUserTelegramChatID].GetValueInt()
|
||||
if err != nil {
|
||||
displayConfig.FormNotificationTelegramChatIDValidation = &libtemplate.FormValidation{
|
||||
Valid: false,
|
||||
Response: "Chat ID must be integer.",
|
||||
}
|
||||
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
|
||||
return valid, nil
|
||||
}
|
||||
@ -5,23 +5,37 @@ import (
|
||||
"github.com/feditools/go-lib/language"
|
||||
"github.com/feditools/relay/internal/http/template"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func (m *Module) executeTemplate(w http.ResponseWriter, name string, tmplVars interface{}) error {
|
||||
func (m *Module) executeTemplate(w http.ResponseWriter, name string, tmplVars interface{}) {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "executeTemplate",
|
||||
"template": name,
|
||||
})
|
||||
|
||||
b := new(bytes.Buffer)
|
||||
err := m.templates.ExecuteTemplate(b, name, tmplVars)
|
||||
if err != nil {
|
||||
return err
|
||||
l.Errorf("could not render template: %s", err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if m.minify == nil {
|
||||
_, err := w.Write(b.Bytes())
|
||||
_, err = w.Write(b.Bytes())
|
||||
if err != nil {
|
||||
l.Errorf("could write response to body: %s", err.Error())
|
||||
}
|
||||
|
||||
return err
|
||||
return
|
||||
}
|
||||
|
||||
return m.minify.Minify("text/html", w, b)
|
||||
err = m.minify.Minify("text/html", w, b)
|
||||
if err != nil {
|
||||
l.Errorf("could write minified response to body: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) initTemplate(_ http.ResponseWriter, r *http.Request, tmpl template.InitTemplate) error {
|
||||
@ -35,9 +49,6 @@ func (m *Module) initTemplate(_ http.ResponseWriter, r *http.Request, tmpl templ
|
||||
lang := r.Context().Value(ContextKeyLanguage).(string) // nolint
|
||||
tmpl.SetLanguage(lang)
|
||||
|
||||
// set logo image src
|
||||
tmpl.SetLogoSrc(m.logoSrcDark, m.logoSrcLight)
|
||||
|
||||
// add css
|
||||
for _, link := range m.headLinks {
|
||||
tmpl.AddHeadLink(link)
|
||||
|
||||
@ -15,10 +15,10 @@ import (
|
||||
// auth helpers
|
||||
|
||||
func (m *Module) authRequireLoggedIn(w nethttp.ResponseWriter, r *nethttp.Request) (*models.Account, bool) {
|
||||
us := r.Context().Value(ContextKeySession).(*sessions.Session) // nolint
|
||||
|
||||
account, ok := r.Context().Value(ContextKeyAccount).(*models.Account)
|
||||
if !ok {
|
||||
us := r.Context().Value(ContextKeySession).(*sessions.Session) // nolint
|
||||
|
||||
// Save current page
|
||||
if r.URL.Query().Encode() == "" {
|
||||
us.Values[SessionKeyLoginRedirect] = r.URL.Path
|
||||
|
||||
@ -14,7 +14,9 @@ type Logic interface {
|
||||
DeliverActivity(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error
|
||||
Domain() string
|
||||
GetBlockList(ctx context.Context) (*[]string, error)
|
||||
GetAccountConfigMap(ctx context.Context, keys ...models.ConfigKey) (*models.AccountConfigMap, error)
|
||||
GetConfigMap(ctx context.Context, keys ...models.ConfigKey) (*models.ConfigMap, error)
|
||||
GetConfigMapForAccount(ctx context.Context, accountID int64, keys ...models.ConfigKey) (*models.ConfigMap, error)
|
||||
GetInstance(ctx context.Context, domain string) (*models.Instance, error)
|
||||
GetInstanceForActor(ctx context.Context, actorID *url.URL) (*models.Instance, error)
|
||||
GetInstanceForServerHostname(ctx context.Context, serverHostname string) (*models.Instance, error)
|
||||
@ -27,6 +29,7 @@ type Logic interface {
|
||||
ProcessBlockDelete(ctx context.Context, blockID int64) error
|
||||
ProcessBlockUpdate(ctx context.Context, blockID int64) error
|
||||
ProcessConfigChanges(ctx context.Context, configChanges []*models.ConfigChange) error
|
||||
SendNotification(ctx context.Context, jid string, event models.EventType, metadata map[string]interface{}) error
|
||||
UpdateBlock(ctx context.Context, blocks ...*models.Block) error
|
||||
ValidateRequest(r *http.Request, actorURI *url.URL) (bool, *fedihelper.Actor)
|
||||
}
|
||||
|
||||
@ -7,10 +7,12 @@ import (
|
||||
"fmt"
|
||||
"github.com/feditools/go-lib/fedihelper"
|
||||
libhttp "github.com/feditools/go-lib/http"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/feditools/relay/internal/path"
|
||||
"github.com/google/uuid"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func (l *Logic) DeliverActivity(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error {
|
||||
@ -70,13 +72,13 @@ func (l *Logic) ProcessActivity(ctx context.Context, jid string, instanceID int6
|
||||
|
||||
switch actType {
|
||||
case fedihelper.TypeAnnounce, fedihelper.TypeCreate:
|
||||
return l.doRelay(ctx, instanceID, activity)
|
||||
return l.doRelay(ctx, jid, instanceID, activity)
|
||||
case fedihelper.TypeDelete, fedihelper.TypeUpdate:
|
||||
return l.doForward(ctx, instanceID, activity)
|
||||
return l.doForward(ctx, jid, instanceID, activity)
|
||||
case fedihelper.TypeFollow:
|
||||
return l.doFollow(ctx, instanceID, activity)
|
||||
return l.doFollow(ctx, jid, instanceID, activity)
|
||||
case fedihelper.TypeUndo:
|
||||
return l.doUndo(ctx, instanceID, activity)
|
||||
return l.doUndo(ctx, jid, instanceID, activity)
|
||||
default:
|
||||
log.Debugf("unhandled activity type: %s", actType)
|
||||
|
||||
@ -84,9 +86,10 @@ func (l *Logic) ProcessActivity(ctx context.Context, jid string, instanceID int6
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Logic) doFollow(ctx context.Context, instanceID int64, activity fedihelper.Activity) error {
|
||||
func (l *Logic) doFollow(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error {
|
||||
log := logger.WithFields(logrus.Fields{
|
||||
"func": "doFollow",
|
||||
"jid": jid,
|
||||
})
|
||||
log.Trace("doFollow called")
|
||||
|
||||
@ -143,12 +146,21 @@ func (l *Logic) doFollow(ctx context.Context, instanceID int64, activity fedihel
|
||||
return fmt.Errorf("can't post to instance: %s\n%s", err.Error(), resp)
|
||||
}
|
||||
|
||||
go l.runner.EnqueueSendNotification(
|
||||
ctx,
|
||||
models.EventInstanceFollow,
|
||||
map[string]interface{}{
|
||||
"instance": strconv.FormatInt(instance.ID, 10),
|
||||
},
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Logic) doForward(ctx context.Context, instanceID int64, activity fedihelper.Activity) error {
|
||||
func (l *Logic) doForward(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error {
|
||||
log := logger.WithFields(logrus.Fields{
|
||||
"func": "doForward",
|
||||
"jid": jid,
|
||||
})
|
||||
log.Trace("doForward called")
|
||||
|
||||
@ -199,9 +211,10 @@ func (l *Logic) doForward(ctx context.Context, instanceID int64, activity fedihe
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Logic) doRelay(ctx context.Context, instanceID int64, activity fedihelper.Activity) error {
|
||||
func (l *Logic) doRelay(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error {
|
||||
log := logger.WithFields(logrus.Fields{
|
||||
"func": "doRelay",
|
||||
"jid": jid,
|
||||
})
|
||||
log.Trace("doRelay called")
|
||||
|
||||
@ -228,7 +241,7 @@ func (l *Logic) doRelay(ctx context.Context, instanceID int64, activity fedihelp
|
||||
}
|
||||
|
||||
// forward activity
|
||||
log.Debugf("relaying post from %s", signingInstance.ActorIRI)
|
||||
log.Infof("relaying post from %s", signingInstance.ActorIRI)
|
||||
log.Tracef("relaying activity: %#v", activity)
|
||||
|
||||
// send announce
|
||||
@ -261,9 +274,10 @@ func (l *Logic) doRelay(ctx context.Context, instanceID int64, activity fedihelp
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Logic) doUndo(ctx context.Context, instanceID int64, activity fedihelper.Activity) error {
|
||||
func (l *Logic) doUndo(ctx context.Context, jid string, instanceID int64, activity fedihelper.Activity) error {
|
||||
log := logger.WithFields(logrus.Fields{
|
||||
"func": "doUndo",
|
||||
"jid": jid,
|
||||
})
|
||||
log.Trace("doUndo called")
|
||||
|
||||
@ -274,7 +288,7 @@ func (l *Logic) doUndo(ctx context.Context, instanceID int64, activity fedihelpe
|
||||
|
||||
switch aType {
|
||||
case fedihelper.TypeAnnounce:
|
||||
return l.doForward(ctx, instanceID, activity)
|
||||
return l.doForward(ctx, jid, instanceID, activity)
|
||||
case fedihelper.TypeFollow:
|
||||
// get instance
|
||||
instance, err := l.db.ReadInstance(ctx, instanceID)
|
||||
@ -317,6 +331,14 @@ func (l *Logic) doUndo(ctx context.Context, instanceID int64, activity fedihelpe
|
||||
return fmt.Errorf("can't post to instance: %s\n%s", err.Error(), resp)
|
||||
}
|
||||
|
||||
go l.runner.EnqueueSendNotification(
|
||||
ctx,
|
||||
models.EventInstanceUnfollow,
|
||||
map[string]interface{}{
|
||||
"instance": strconv.FormatInt(instance.ID, 10),
|
||||
},
|
||||
)
|
||||
|
||||
return nil
|
||||
default:
|
||||
log.Debugf("dropping activity object type: %s", aType)
|
||||
|
||||
@ -7,13 +7,63 @@ import (
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
func (l *Logic) GetAccountConfigMap(ctx context.Context, keys ...models.ConfigKey) (*models.AccountConfigMap, error) {
|
||||
accountConfigMap := models.AccountConfigMap{}
|
||||
|
||||
configs, err := l.db.ReadConfigsForKeysUser(ctx, keys...)
|
||||
if err != nil {
|
||||
if errors.Is(err, db.ErrNoEntries) {
|
||||
return &accountConfigMap, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, config := range configs {
|
||||
// create entry for account
|
||||
_, exists := accountConfigMap[config.AccountID]
|
||||
if !exists {
|
||||
accountConfigMap[config.AccountID] = models.ConfigMap{}
|
||||
for _, key := range keys {
|
||||
accountConfigMap[config.AccountID][key] = nil
|
||||
}
|
||||
}
|
||||
|
||||
accountConfigMap[config.AccountID][config.Key] = config
|
||||
}
|
||||
|
||||
return &accountConfigMap, nil
|
||||
}
|
||||
|
||||
func (l *Logic) GetConfigMap(ctx context.Context, keys ...models.ConfigKey) (*models.ConfigMap, error) {
|
||||
configMap := models.ConfigMap{}
|
||||
for _, key := range keys {
|
||||
configMap[key] = nil
|
||||
}
|
||||
|
||||
configs, err := l.db.ReadConfigsForKeys(ctx, keys...)
|
||||
configs, err := l.db.ReadConfigsForKeysSystem(ctx, keys...)
|
||||
if err != nil {
|
||||
if errors.Is(err, db.ErrNoEntries) {
|
||||
return &configMap, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, config := range configs {
|
||||
configMap[config.Key] = config
|
||||
}
|
||||
|
||||
return &configMap, nil
|
||||
}
|
||||
|
||||
func (l *Logic) GetConfigMapForAccount(ctx context.Context, accountID int64, keys ...models.ConfigKey) (*models.ConfigMap, error) {
|
||||
configMap := models.ConfigMap{}
|
||||
for _, key := range keys {
|
||||
configMap[key] = nil
|
||||
}
|
||||
|
||||
configs, err := l.db.ReadConfigsForAccountForKeys(ctx, accountID, keys...)
|
||||
if err != nil {
|
||||
if errors.Is(err, db.ErrNoEntries) {
|
||||
return &configMap, nil
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"github.com/feditools/relay/internal/db"
|
||||
"github.com/feditools/relay/internal/fedi"
|
||||
ihttp "github.com/feditools/relay/internal/http"
|
||||
"github.com/feditools/relay/internal/notification"
|
||||
"github.com/feditools/relay/internal/path"
|
||||
"github.com/feditools/relay/internal/runner"
|
||||
"github.com/feditools/relay/internal/token"
|
||||
@ -24,6 +25,7 @@ import (
|
||||
type Logic struct {
|
||||
db db.DB
|
||||
fedi *fedi.Module
|
||||
notifier notification.Notifier
|
||||
runner runner.Runner
|
||||
tokz *token.Tokenizer
|
||||
transport *fedihelper.Transport
|
||||
@ -116,6 +118,10 @@ func (l *Logic) SetFedi(f *fedi.Module) {
|
||||
l.fedi = f
|
||||
}
|
||||
|
||||
func (l *Logic) SetNotifier(n notification.Notifier) {
|
||||
l.notifier = n
|
||||
}
|
||||
|
||||
func (l *Logic) SetRunner(r runner.Runner) {
|
||||
l.runner = r
|
||||
}
|
||||
|
||||
84
internal/logic/logic1/notification.go
Normal file
84
internal/logic/logic1/notification.go
Normal file
@ -0,0 +1,84 @@
|
||||
package logic1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/sirupsen/logrus"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func (l *Logic) SendNotification(ctx context.Context, jid string, event models.EventType, metadata map[string]interface{}) error {
|
||||
switch event {
|
||||
case models.EventInstanceFollow:
|
||||
return l.doNotificationInstanceFollow(ctx, jid, metadata)
|
||||
case models.EventInstanceUnfollow:
|
||||
return l.doNotificationInstanceUnfollow(ctx, jid, metadata)
|
||||
default:
|
||||
logger.WithFields(logrus.Fields{
|
||||
"func": "sendNotification",
|
||||
"jid": jid,
|
||||
}).Warnf("unrecognized action %s", event)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Logic) doNotificationInstanceFollow(ctx context.Context, jid string, metadata map[string]interface{}) error {
|
||||
return l.helperNotificationInstance(ctx, jid, metadata, "The instance '%s' has joined the relay.")
|
||||
}
|
||||
|
||||
func (l *Logic) doNotificationInstanceUnfollow(ctx context.Context, jid string, metadata map[string]interface{}) error {
|
||||
return l.helperNotificationInstance(ctx, jid, metadata, "The instance '%s' has left the relay.")
|
||||
}
|
||||
|
||||
func (l *Logic) helperNotificationInstance(ctx context.Context, jid string, metadata map[string]interface{}, messageTemplate string) error {
|
||||
log := logger.WithFields(logrus.Fields{
|
||||
"func": "helperNotificationInstance",
|
||||
"jid": jid,
|
||||
})
|
||||
log.Trace("called")
|
||||
|
||||
// get instance
|
||||
instanceID, err := instanceFromMetadata(&metadata)
|
||||
if err != nil {
|
||||
log.Errorf("instance from metadata: %s", err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
instance, err := l.db.ReadInstance(ctx, int64(instanceID))
|
||||
if err != nil {
|
||||
log.Errorf("db read: %s", err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf(messageTemplate, instance.Domain)
|
||||
log.Debugf("sending notification \"%s\"", msg)
|
||||
err = l.notifier.SendNotification(ctx, msg)
|
||||
if err != nil {
|
||||
log.Errorf("sending notification: %s", err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceFromMetadata(metadata *map[string]interface{}) (int64, error) {
|
||||
instanceIDI, ok := (*metadata)["instance"]
|
||||
if !ok {
|
||||
return 0, errors.New("missing field 'instance'")
|
||||
}
|
||||
instanceIDStr, ok := instanceIDI.(string)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("field 'instance' is not type string, got %T", instanceIDI)
|
||||
}
|
||||
instanceID, err := strconv.ParseInt(instanceIDStr, 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return instanceID, nil
|
||||
}
|
||||
@ -1,8 +1,6 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -22,29 +20,3 @@ type Account struct {
|
||||
IsAdmin bool `validate:"-" bun:",notnull"`
|
||||
IsMod bool `validate:"-" bun:",notnull"`
|
||||
}
|
||||
|
||||
var _ bun.BeforeAppendModelHook = (*Account)(nil)
|
||||
|
||||
// BeforeAppendModel runs before a bun append operation.
|
||||
func (a *Account) BeforeAppendModel(_ context.Context, query bun.Query) error {
|
||||
switch query.(type) {
|
||||
case *bun.InsertQuery:
|
||||
now := time.Now()
|
||||
a.CreatedAt = now
|
||||
a.UpdatedAt = now
|
||||
|
||||
err := validate.Struct(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *bun.UpdateQuery:
|
||||
a.UpdatedAt = time.Now()
|
||||
|
||||
err := validate.Struct(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -16,28 +14,3 @@ type Block struct {
|
||||
ObfuscatedDomain string `validate:"-" bun:",nullzero" json:"obfuscated_domain,omitempty"`
|
||||
BlockSubdomains bool `validate:"-" bun:",notnull" json:"block_subdomains"`
|
||||
}
|
||||
|
||||
var _ bun.BeforeAppendModelHook = (*Block)(nil)
|
||||
|
||||
// BeforeAppendModel runs before a bun append operation
|
||||
func (b *Block) BeforeAppendModel(_ context.Context, query bun.Query) error {
|
||||
switch query.(type) {
|
||||
case *bun.InsertQuery:
|
||||
now := time.Now()
|
||||
b.CreatedAt = now
|
||||
b.UpdatedAt = now
|
||||
|
||||
err := validate.Struct(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *bun.UpdateQuery:
|
||||
b.UpdatedAt = time.Now()
|
||||
|
||||
err := validate.Struct(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,10 +1,9 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"github.com/feditools/relay/internal/util"
|
||||
"github.com/uptrace/bun"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@ -17,32 +16,8 @@ type Config struct {
|
||||
Key ConfigKey `validate:"-" bun:",unique,nullzero,notnull"`
|
||||
Value string `validate:"-" bun:",nullzero"`
|
||||
IsEncrypted bool `validate:"-" bun:",notnull"`
|
||||
}
|
||||
|
||||
var _ bun.BeforeAppendModelHook = (*Config)(nil)
|
||||
|
||||
// BeforeAppendModel runs before a bun append operation.
|
||||
func (c *Config) BeforeAppendModel(_ context.Context, query bun.Query) error {
|
||||
switch query.(type) {
|
||||
case *bun.InsertQuery:
|
||||
now := time.Now()
|
||||
c.CreatedAt = now
|
||||
c.UpdatedAt = now
|
||||
|
||||
err := validate.Struct(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *bun.UpdateQuery:
|
||||
c.UpdatedAt = time.Now()
|
||||
|
||||
err := validate.Struct(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
AccountID int64 `validate:"-" bun:",nullzero"`
|
||||
Account *Account `validate:"-" bun:"rel:belongs-to,join:account_id=id"`
|
||||
}
|
||||
|
||||
func (c *Config) GetValue() (string, error) {
|
||||
@ -60,6 +35,15 @@ func (c *Config) GetValue() (string, error) {
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func (c *Config) GetValueBool() (bool, error) {
|
||||
value, err := c.GetValue()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return strconv.ParseBool(value)
|
||||
}
|
||||
|
||||
func (c *Config) GetValueInt() (int, error) {
|
||||
value, err := c.GetValue()
|
||||
if err != nil {
|
||||
@ -74,10 +58,11 @@ func (c *Config) GetValueInt() (int, error) {
|
||||
return int(number), nil
|
||||
}
|
||||
|
||||
func (c *Config) SetValue(value string) error {
|
||||
func (c *Config) SetValue(value string, encrypt bool) error {
|
||||
// not encrypted, set raw value
|
||||
if !configKeyEncrypted[c.Key] {
|
||||
if !encrypt {
|
||||
c.Value = value
|
||||
c.IsEncrypted = false
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -86,10 +71,15 @@ func (c *Config) SetValue(value string) error {
|
||||
return err
|
||||
}
|
||||
c.Value = base64.StdEncoding.EncodeToString(encryptedValue)
|
||||
c.IsEncrypted = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Config) SetValueInt(value int) error {
|
||||
return c.SetValue(strconv.FormatInt(int64(value), 10))
|
||||
func (c *Config) SetValueBool(value bool, encrypt bool) error {
|
||||
return c.SetValue(fmt.Sprintf("%t", value), encrypt)
|
||||
}
|
||||
|
||||
func (c *Config) SetValueInt(value int, encrypt bool) error {
|
||||
return c.SetValue(strconv.FormatInt(int64(value), 10), encrypt)
|
||||
}
|
||||
|
||||
@ -3,19 +3,13 @@ package models
|
||||
type ConfigKey string
|
||||
|
||||
const (
|
||||
ConfigHomeBody ConfigKey = "home.body"
|
||||
ConfigTelegramToken ConfigKey = "telegram.token"
|
||||
ConfigHomeBody ConfigKey = `home.body`
|
||||
ConfigTelegramEnabled ConfigKey = `telegram.enabled`
|
||||
ConfigTelegramToken ConfigKey = `telegram.token`
|
||||
ConfigUserTelegramEnabled ConfigKey = `user.telegram.enabled`
|
||||
ConfigUserTelegramChatID ConfigKey = `user.telegram.chat_id`
|
||||
)
|
||||
|
||||
var configKeyEncrypted = map[ConfigKey]bool{
|
||||
ConfigHomeBody: false,
|
||||
ConfigTelegramToken: true,
|
||||
}
|
||||
|
||||
func (k ConfigKey) String() string {
|
||||
return string(k)
|
||||
}
|
||||
|
||||
func (k ConfigKey) Encrypted() bool {
|
||||
return configKeyEncrypted[k]
|
||||
}
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
package models
|
||||
|
||||
type ConfigMap map[ConfigKey]*Config
|
||||
|
||||
type AccountConfigMap map[int64]ConfigMap
|
||||
|
||||
8
internal/models/event.go
Normal file
8
internal/models/event.go
Normal file
@ -0,0 +1,8 @@
|
||||
package models
|
||||
|
||||
type EventType string
|
||||
|
||||
const (
|
||||
EventInstanceFollow EventType = "instance_follow"
|
||||
EventInstanceUnfollow EventType = "instance_unfollow"
|
||||
)
|
||||
@ -2,14 +2,12 @@ package models
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"github.com/feditools/relay/internal/util"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/uptrace/bun"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
@ -33,31 +31,6 @@ type Instance struct {
|
||||
Block *Block `validate:"-" bun:"rel:belongs-to"`
|
||||
}
|
||||
|
||||
var _ bun.BeforeAppendModelHook = (*Instance)(nil)
|
||||
|
||||
// BeforeAppendModel runs before a bun append operation
|
||||
func (i *Instance) BeforeAppendModel(_ context.Context, query bun.Query) error {
|
||||
switch query.(type) {
|
||||
case *bun.InsertQuery:
|
||||
now := time.Now()
|
||||
i.CreatedAt = now
|
||||
i.UpdatedAt = now
|
||||
|
||||
err := validate.Struct(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *bun.UpdateQuery:
|
||||
i.UpdatedAt = time.Now()
|
||||
|
||||
err := validate.Struct(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetOAuthClientSecret returns unencrypted client secret.
|
||||
func (i *Instance) GetOAuthClientSecret() (string, error) {
|
||||
encryptedData, err := base64.StdEncoding.DecodeString(i.OAuthClientSecret)
|
||||
|
||||
7
internal/models/notification_service.go
Normal file
7
internal/models/notification_service.go
Normal file
@ -0,0 +1,7 @@
|
||||
package models
|
||||
|
||||
type NotificationService string
|
||||
|
||||
const (
|
||||
ServiceTelegram NotificationService = "telegram"
|
||||
)
|
||||
@ -1,9 +0,0 @@
|
||||
package models
|
||||
|
||||
import "github.com/go-playground/validator/v10"
|
||||
|
||||
var validate *validator.Validate
|
||||
|
||||
func init() {
|
||||
validate = validator.New()
|
||||
}
|
||||
9
internal/notification/manager/logger.go
Normal file
9
internal/notification/manager/logger.go
Normal file
@ -0,0 +1,9 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/log"
|
||||
)
|
||||
|
||||
type empty struct{}
|
||||
|
||||
var logger = log.WithPackageField(empty{})
|
||||
20
internal/notification/manager/manager.go
Normal file
20
internal/notification/manager/manager.go
Normal file
@ -0,0 +1,20 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/feditools/relay/internal/notification"
|
||||
)
|
||||
|
||||
func New() (*Manager, error) {
|
||||
return &Manager{
|
||||
services: map[models.NotificationService]notification.Service{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
services map[models.NotificationService]notification.Service
|
||||
}
|
||||
|
||||
func (m *Manager) AddService(service notification.Service) {
|
||||
m.services[service.GetService()] = service
|
||||
}
|
||||
23
internal/notification/manager/notification.go
Normal file
23
internal/notification/manager/notification.go
Normal file
@ -0,0 +1,23 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (m *Manager) SendNotification(ctx context.Context, message string) error {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "SendNotification",
|
||||
})
|
||||
|
||||
for _, service := range m.services {
|
||||
err := service.SendNotification(ctx, message)
|
||||
if err != nil {
|
||||
l.Errorf("sending notification with %s: %s", service.GetService(), err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
25
internal/notification/notifier.go
Normal file
25
internal/notification/notifier.go
Normal file
@ -0,0 +1,25 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
// Notifier sends notifications.
|
||||
type Notifier interface {
|
||||
SendNotification(ctx context.Context, message string) error
|
||||
}
|
||||
|
||||
// Manager sends notifications to multiple sources
|
||||
type Manager interface {
|
||||
Notifier
|
||||
|
||||
AddService(service Service)
|
||||
}
|
||||
|
||||
// Service sends notifications to a service
|
||||
type Service interface {
|
||||
Notifier
|
||||
|
||||
GetService() models.NotificationService
|
||||
}
|
||||
41
internal/notification/telegram/client.go
Normal file
41
internal/notification/telegram/client.go
Normal file
@ -0,0 +1,41 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/nickname76/telegrambot"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (s *Service) newClient(ctx context.Context) (*telegrambot.API, error) {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "newClient",
|
||||
})
|
||||
|
||||
// get config
|
||||
cm, err := s.logic.GetConfigMap(ctx, models.ConfigTelegramToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// return error if missing config
|
||||
if (*cm)[models.ConfigTelegramToken] == nil {
|
||||
return nil, fmt.Errorf("config %s missing", models.ConfigTelegramToken)
|
||||
}
|
||||
|
||||
// create api
|
||||
token, err := (*cm)[models.ConfigTelegramToken].GetValue()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l.Tracef("got token: %s", token)
|
||||
api, me, err := telegrambot.NewAPI(token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l.Debugf("authorized on account %s", me.Username)
|
||||
|
||||
return api, nil
|
||||
}
|
||||
37
internal/notification/telegram/config.go
Normal file
37
internal/notification/telegram/config.go
Normal file
@ -0,0 +1,37 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
func parseConfigMap(cm *models.ConfigMap) (bool, int, error) {
|
||||
// return not enabled if no telegram enabled config set
|
||||
if (*cm)[models.ConfigUserTelegramEnabled] == nil {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
// get enabled value
|
||||
enabled, err := (*cm)[models.ConfigUserTelegramEnabled].GetValueBool()
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
|
||||
// return not enabled if not enabled
|
||||
if !enabled {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
// return error if no telegram enabled config set
|
||||
if (*cm)[models.ConfigUserTelegramChatID] == nil {
|
||||
return false, 0, fmt.Errorf("config %s missing", models.ConfigUserTelegramChatID)
|
||||
}
|
||||
|
||||
// get chat id
|
||||
chatID, err := (*cm)[models.ConfigUserTelegramChatID].GetValueInt()
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
|
||||
return true, chatID, nil
|
||||
}
|
||||
19
internal/notification/telegram/enabled.go
Normal file
19
internal/notification/telegram/enabled.go
Normal file
@ -0,0 +1,19 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
func (s *Service) isEnabled(ctx context.Context) (bool, error) {
|
||||
cm, err := s.logic.GetConfigMap(ctx, models.ConfigTelegramEnabled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if (*cm)[models.ConfigTelegramEnabled] != nil {
|
||||
return (*cm)[models.ConfigTelegramEnabled].GetValueBool()
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
9
internal/notification/telegram/logger.go
Normal file
9
internal/notification/telegram/logger.go
Normal file
@ -0,0 +1,9 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/log"
|
||||
)
|
||||
|
||||
type empty struct{}
|
||||
|
||||
var logger = log.WithPackageField(empty{})
|
||||
130
internal/notification/telegram/notification.go
Normal file
130
internal/notification/telegram/notification.go
Normal file
@ -0,0 +1,130 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/nickname76/telegrambot"
|
||||
"github.com/sirupsen/logrus"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (s *Service) SendNotification(ctx context.Context, message string) error {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "SendNotification",
|
||||
})
|
||||
|
||||
enabled, err := s.isEnabled(ctx)
|
||||
if err != nil {
|
||||
l.Errorf("db read: %s", err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
if enabled {
|
||||
return s.sendNotification(ctx, message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) sendNotification(ctx context.Context, message string) error {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "sendNotification",
|
||||
})
|
||||
|
||||
client, err := s.newClient(ctx)
|
||||
if err != nil {
|
||||
l.Errorf("creating client: %s", err.Error())
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
acm, err := s.logic.GetAccountConfigMap(
|
||||
ctx,
|
||||
models.ConfigUserTelegramEnabled,
|
||||
models.ConfigUserTelegramChatID,
|
||||
)
|
||||
if err != nil {
|
||||
l.Errorf("db read: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
var errorList []error
|
||||
for account, cm := range *acm {
|
||||
enabled, chatID, err := parseConfigMap(&cm)
|
||||
if err != nil {
|
||||
errorList = append(errorList, err)
|
||||
continue
|
||||
}
|
||||
l.Tracef("got cm for %d: %+v", account, cm)
|
||||
|
||||
if enabled && shouldSend(cm[models.ConfigUserTelegramChatID].Account) {
|
||||
err = doSend(ctx, client, telegrambot.ChatID(chatID), message)
|
||||
if err != nil {
|
||||
errorList = append(errorList, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch len(errorList) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
l.Errorf("sending notification: %s", errorList[0].Error())
|
||||
|
||||
return errorList[0]
|
||||
default:
|
||||
lastErr := len(errorList) - 1
|
||||
var errStr strings.Builder
|
||||
for i, e := range errorList {
|
||||
errStr.WriteString(e.Error())
|
||||
|
||||
if i < lastErr {
|
||||
errStr.WriteString("; ")
|
||||
}
|
||||
}
|
||||
l.Errorf("sending notifications: %s", errStr.String())
|
||||
|
||||
return errors.New(errStr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func doSend(_ context.Context, client *telegrambot.API, chatID telegrambot.ChatIDOrUsername, message string) error {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "doSend",
|
||||
})
|
||||
|
||||
tgMsgParams := &telegrambot.SendMessageParams{
|
||||
ChatID: chatID,
|
||||
Text: message,
|
||||
}
|
||||
|
||||
l.Tracef("sending: %+v", tgMsgParams)
|
||||
_, err := client.SendMessage(tgMsgParams)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func shouldSend(account *models.Account) bool {
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "shouldSend",
|
||||
})
|
||||
|
||||
// account deleted
|
||||
if account == nil {
|
||||
l.Tracef("false: account is nil")
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// send for admins and mods
|
||||
if account.IsAdmin || account.IsMod {
|
||||
l.Tracef("true: account is admin or mod")
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
l.Tracef("false")
|
||||
return false
|
||||
}
|
||||
20
internal/notification/telegram/telegram.go
Normal file
20
internal/notification/telegram/telegram.go
Normal file
@ -0,0 +1,20 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"github.com/feditools/relay/internal/logic"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
func New(l logic.Logic) (*Service, error) {
|
||||
return &Service{
|
||||
logic: l,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
logic logic.Logic
|
||||
}
|
||||
|
||||
func (*Service) GetService() models.NotificationService {
|
||||
return models.ServiceTelegram
|
||||
}
|
||||
@ -42,6 +42,11 @@ func GenAppLogoutPath() string {
|
||||
return AppLogout
|
||||
}
|
||||
|
||||
// GenAppSettingsHomePath returns a path for the settings page.
|
||||
func GenAppSettingsHomePath() string {
|
||||
return AppSettingsHome
|
||||
}
|
||||
|
||||
// urls
|
||||
|
||||
// GenActor returns a url for an actor
|
||||
|
||||
@ -31,6 +31,8 @@ const (
|
||||
PartOauth = "oauth"
|
||||
// PartPublicKey is the noun used in a path for an actor's public key
|
||||
PartPublicKey = "main-key"
|
||||
// PartSettings is used in a path for settings page.
|
||||
PartSettings = "settings"
|
||||
// PartStatic is used in a path for static files.
|
||||
PartStatic = "static"
|
||||
// PartWebFinger is the noun used in a path for web finger
|
||||
|
||||
@ -26,6 +26,14 @@ const (
|
||||
AppLogout = App + AppSubLogout
|
||||
// AppSubLogout is the sub path for the logout page.
|
||||
AppSubLogout = "/" + PartLogout
|
||||
// AppSettings is the path for the settings page.
|
||||
AppSettings = App + AppSubSettings
|
||||
// AppSubSettings is the sub path for the settings page.
|
||||
AppSubSettings = "/" + PartSettings
|
||||
// AppSettingsHome is the path for the home page.
|
||||
AppSettingsHome = AppSettings + AppSettingsSubHome
|
||||
// AppSettingsSubHome is the sub path for the home page.
|
||||
AppSettingsSubHome = "/"
|
||||
|
||||
// admin
|
||||
|
||||
|
||||
@ -6,8 +6,9 @@ const (
|
||||
JobProcessBlockAdd = "ProcessBlockAdd"
|
||||
JobProcessBlockDelete = "ProcessBlockDelete"
|
||||
JobProcessBlockUpdate = "ProcessBlockUpdate"
|
||||
JobSendNotification = "SendNotification"
|
||||
|
||||
QueueDefault = "default"
|
||||
QueueDelivery = "delivery"
|
||||
QueuePriority = "priority"
|
||||
QueueDefault = "default" // medium
|
||||
QueueDelivery = "delivery" // low
|
||||
QueuePriority = "priority" // high
|
||||
)
|
||||
|
||||
50
internal/runner/faktory/notification.go
Normal file
50
internal/runner/faktory/notification.go
Normal file
@ -0,0 +1,50 @@
|
||||
package faktory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
faktory "github.com/contribsys/faktory/client"
|
||||
worker "github.com/contribsys/faktory_worker_go"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (r *Runner) EnqueueSendNotification(_ context.Context, event models.EventType, metadata map[string]interface{}) error {
|
||||
job := faktory.NewJob(JobSendNotification, event, metadata)
|
||||
job.Queue = QueuePriority
|
||||
|
||||
client, err := r.manager.Pool.Get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return client.Push(job)
|
||||
}
|
||||
|
||||
func (r *Runner) sendNotification(ctx context.Context, args ...interface{}) error {
|
||||
help := worker.HelperFor(ctx)
|
||||
|
||||
l := logger.WithFields(logrus.Fields{
|
||||
"func": "sendNotification",
|
||||
"jid": help.Jid(),
|
||||
})
|
||||
|
||||
if len(args) != 2 {
|
||||
l.Errorf("wrong number of arguments, got: %d, want: %d", len(args), 2)
|
||||
}
|
||||
|
||||
// cast arguments
|
||||
event, ok := args[0].(string)
|
||||
if !ok {
|
||||
l.Errorf("argument 0 is not an string")
|
||||
|
||||
return fmt.Errorf("argument 0 is not an int")
|
||||
}
|
||||
metadata, ok := args[1].(map[string]interface{})
|
||||
if !ok {
|
||||
l.Errorf("argument 1 is not an activity, got %T", args[2])
|
||||
|
||||
return fmt.Errorf("argument 1 is not an activity")
|
||||
}
|
||||
|
||||
return r.logic.SendNotification(ctx, help.Jid(), models.EventType(event), metadata)
|
||||
}
|
||||
@ -30,6 +30,7 @@ func New(l logic.Logic) (*Runner, error) {
|
||||
mgr.Register(JobProcessBlockAdd, newRunner.processBlockAdd)
|
||||
mgr.Register(JobProcessBlockDelete, newRunner.processBlockDelete)
|
||||
mgr.Register(JobProcessBlockUpdate, newRunner.processBlockUpdate)
|
||||
mgr.Register(JobSendNotification, newRunner.sendNotification)
|
||||
|
||||
newRunner.manager = mgr
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package runner
|
||||
import (
|
||||
"context"
|
||||
"github.com/feditools/go-lib/fedihelper"
|
||||
"github.com/feditools/relay/internal/models"
|
||||
)
|
||||
|
||||
type Runner interface {
|
||||
@ -11,4 +12,5 @@ type Runner interface {
|
||||
EnqueueProcessBlockAdd(ctx context.Context, blockID int64) (err error)
|
||||
EnqueueProcessBlockDelete(ctx context.Context, blockID int64) (err error)
|
||||
EnqueueProcessBlockUpdate(ctx context.Context, blockID int64) (err error)
|
||||
EnqueueSendNotification(ctx context.Context, event models.EventType, metadata map[string]interface{}) (err error)
|
||||
}
|
||||
|
||||
19
vendor/github.com/andybalholm/brotli/LICENSE
generated
vendored
Normal file
19
vendor/github.com/andybalholm/brotli/LICENSE
generated
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
7
vendor/github.com/andybalholm/brotli/README.md
generated
vendored
Normal file
7
vendor/github.com/andybalholm/brotli/README.md
generated
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
This package is a brotli compressor and decompressor implemented in Go.
|
||||
It was translated from the reference implementation (https://github.com/google/brotli)
|
||||
with the `c2go` tool at https://github.com/andybalholm/c2go.
|
||||
|
||||
I am using it in production with https://github.com/andybalholm/redwood.
|
||||
|
||||
API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc.
|
||||
185
vendor/github.com/andybalholm/brotli/backward_references.go
generated
vendored
Normal file
185
vendor/github.com/andybalholm/brotli/backward_references.go
generated
vendored
Normal file
@ -0,0 +1,185 @@
|
||||
package brotli
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find backward reference copies. */
|
||||
|
||||
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
|
||||
if distance <= max_distance {
|
||||
var distance_plus_3 uint = distance + 3
|
||||
var offset0 uint = distance_plus_3 - uint(dist_cache[0])
|
||||
var offset1 uint = distance_plus_3 - uint(dist_cache[1])
|
||||
if distance == uint(dist_cache[0]) {
|
||||
return 0
|
||||
} else if distance == uint(dist_cache[1]) {
|
||||
return 1
|
||||
} else if offset0 < 7 {
|
||||
return (0x9750468 >> (4 * offset0)) & 0xF
|
||||
} else if offset1 < 7 {
|
||||
return (0xFDB1ACE >> (4 * offset1)) & 0xF
|
||||
} else if distance == uint(dist_cache[2]) {
|
||||
return 2
|
||||
} else if distance == uint(dist_cache[3]) {
|
||||
return 3
|
||||
}
|
||||
}
|
||||
|
||||
return distance + numDistanceShortCodes - 1
|
||||
}
|
||||
|
||||
var hasherSearchResultPool sync.Pool
|
||||
|
||||
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||
var insert_length uint = *last_insert_len
|
||||
var pos_end uint = position + num_bytes
|
||||
var store_end uint
|
||||
if num_bytes >= hasher.StoreLookahead() {
|
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1
|
||||
} else {
|
||||
store_end = position
|
||||
}
|
||||
var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params)
|
||||
var apply_random_heuristics uint = position + random_heuristics_window_size
|
||||
var gap uint = 0
|
||||
/* Set maximum distance, see section 9.1. of the spec. */
|
||||
|
||||
const kMinScore uint = scoreBase + 100
|
||||
|
||||
/* For speed up heuristics for random data. */
|
||||
|
||||
/* Minimum score to accept a backward reference. */
|
||||
hasher.PrepareDistanceCache(dist_cache)
|
||||
sr2, _ := hasherSearchResultPool.Get().(*hasherSearchResult)
|
||||
if sr2 == nil {
|
||||
sr2 = &hasherSearchResult{}
|
||||
}
|
||||
sr, _ := hasherSearchResultPool.Get().(*hasherSearchResult)
|
||||
if sr == nil {
|
||||
sr = &hasherSearchResult{}
|
||||
}
|
||||
|
||||
for position+hasher.HashTypeLength() < pos_end {
|
||||
var max_length uint = pos_end - position
|
||||
var max_distance uint = brotli_min_size_t(position, max_backward_limit)
|
||||
sr.len = 0
|
||||
sr.len_code_delta = 0
|
||||
sr.distance = 0
|
||||
sr.score = kMinScore
|
||||
hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, sr)
|
||||
if sr.score > kMinScore {
|
||||
/* Found a match. Let's look for something even better ahead. */
|
||||
var delayed_backward_references_in_row int = 0
|
||||
max_length--
|
||||
for ; ; max_length-- {
|
||||
var cost_diff_lazy uint = 175
|
||||
if params.quality < minQualityForExtensiveReferenceSearch {
|
||||
sr2.len = brotli_min_size_t(sr.len-1, max_length)
|
||||
} else {
|
||||
sr2.len = 0
|
||||
}
|
||||
sr2.len_code_delta = 0
|
||||
sr2.distance = 0
|
||||
sr2.score = kMinScore
|
||||
max_distance = brotli_min_size_t(position+1, max_backward_limit)
|
||||
hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, sr2)
|
||||
if sr2.score >= sr.score+cost_diff_lazy {
|
||||
/* Ok, let's just write one byte for now and start a match from the
|
||||
next byte. */
|
||||
position++
|
||||
|
||||
insert_length++
|
||||
*sr = *sr2
|
||||
delayed_backward_references_in_row++
|
||||
if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size
|
||||
max_distance = brotli_min_size_t(position, max_backward_limit)
|
||||
{
|
||||
/* The first 16 codes are special short-codes,
|
||||
and the minimum offset is 1. */
|
||||
var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache)
|
||||
if (sr.distance <= (max_distance + gap)) && distance_code > 0 {
|
||||
dist_cache[3] = dist_cache[2]
|
||||
dist_cache[2] = dist_cache[1]
|
||||
dist_cache[1] = dist_cache[0]
|
||||
dist_cache[0] = int(sr.distance)
|
||||
hasher.PrepareDistanceCache(dist_cache)
|
||||
}
|
||||
|
||||
*commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code))
|
||||
}
|
||||
|
||||
*num_literals += insert_length
|
||||
insert_length = 0
|
||||
/* Put the hash keys into the table, if there are enough bytes left.
|
||||
Depending on the hasher implementation, it can push all positions
|
||||
in the given range or only a subset of them.
|
||||
Avoid hash poisoning with RLE data. */
|
||||
{
|
||||
var range_start uint = position + 2
|
||||
var range_end uint = brotli_min_size_t(position+sr.len, store_end)
|
||||
if sr.distance < sr.len>>2 {
|
||||
range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2)))
|
||||
}
|
||||
|
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end)
|
||||
}
|
||||
|
||||
position += sr.len
|
||||
} else {
|
||||
insert_length++
|
||||
position++
|
||||
|
||||
/* If we have not seen matches for a long time, we can skip some
|
||||
match lookups. Unsuccessful match lookups are very very expensive
|
||||
and this kind of a heuristic speeds up compression quite
|
||||
a lot. */
|
||||
if position > apply_random_heuristics {
|
||||
/* Going through uncompressible data, jump. */
|
||||
if position > apply_random_heuristics+4*random_heuristics_window_size {
|
||||
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4)
|
||||
/* It is quite a long time since we saw a copy, so we assume
|
||||
that this data is not compressible, and store hashes less
|
||||
often. Hashes of non compressible data are less likely to
|
||||
turn out to be useful in the future, too, so we store less of
|
||||
them to not to flood out the hash table of good compressible
|
||||
data. */
|
||||
|
||||
var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin)
|
||||
for ; position < pos_jump; position += 4 {
|
||||
hasher.Store(ringbuffer, ringbuffer_mask, position)
|
||||
insert_length += 4
|
||||
}
|
||||
} else {
|
||||
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2)
|
||||
var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin)
|
||||
for ; position < pos_jump; position += 2 {
|
||||
hasher.Store(ringbuffer, ringbuffer_mask, position)
|
||||
insert_length += 2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
insert_length += pos_end - position
|
||||
*last_insert_len = insert_length
|
||||
|
||||
hasherSearchResultPool.Put(sr)
|
||||
hasherSearchResultPool.Put(sr2)
|
||||
}
|
||||
796
vendor/github.com/andybalholm/brotli/backward_references_hq.go
generated
vendored
Normal file
796
vendor/github.com/andybalholm/brotli/backward_references_hq.go
generated
vendored
Normal file
@ -0,0 +1,796 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
type zopfliNode struct {
|
||||
length uint32
|
||||
distance uint32
|
||||
dcode_insert_length uint32
|
||||
u struct {
|
||||
cost float32
|
||||
next uint32
|
||||
shortcut uint32
|
||||
}
|
||||
}
|
||||
|
||||
const maxEffectiveDistanceAlphabetSize = 544
|
||||
|
||||
const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
|
||||
|
||||
var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
|
||||
|
||||
var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
|
||||
|
||||
func initZopfliNodes(array []zopfliNode, length uint) {
|
||||
var stub zopfliNode
|
||||
var i uint
|
||||
stub.length = 1
|
||||
stub.distance = 0
|
||||
stub.dcode_insert_length = 0
|
||||
stub.u.cost = kInfinity
|
||||
for i = 0; i < length; i++ {
|
||||
array[i] = stub
|
||||
}
|
||||
}
|
||||
|
||||
func zopfliNodeCopyLength(self *zopfliNode) uint32 {
|
||||
return self.length & 0x1FFFFFF
|
||||
}
|
||||
|
||||
func zopfliNodeLengthCode(self *zopfliNode) uint32 {
|
||||
var modifier uint32 = self.length >> 25
|
||||
return zopfliNodeCopyLength(self) + 9 - modifier
|
||||
}
|
||||
|
||||
func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
|
||||
return self.distance
|
||||
}
|
||||
|
||||
func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
|
||||
var short_code uint32 = self.dcode_insert_length >> 27
|
||||
if short_code == 0 {
|
||||
return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
|
||||
} else {
|
||||
return short_code - 1
|
||||
}
|
||||
}
|
||||
|
||||
func zopfliNodeCommandLength(self *zopfliNode) uint32 {
|
||||
return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
|
||||
}
|
||||
|
||||
/* Histogram based cost model for zopflification. */
|
||||
type zopfliCostModel struct {
|
||||
cost_cmd_ [numCommandSymbols]float32
|
||||
cost_dist_ []float32
|
||||
distance_histogram_size uint32
|
||||
literal_costs_ []float32
|
||||
min_cost_cmd_ float32
|
||||
num_bytes_ uint
|
||||
}
|
||||
|
||||
func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
|
||||
var distance_histogram_size uint32 = dist.alphabet_size
|
||||
if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
|
||||
distance_histogram_size = maxEffectiveDistanceAlphabetSize
|
||||
}
|
||||
|
||||
self.num_bytes_ = num_bytes
|
||||
self.literal_costs_ = make([]float32, (num_bytes + 2))
|
||||
self.cost_dist_ = make([]float32, (dist.alphabet_size))
|
||||
self.distance_histogram_size = distance_histogram_size
|
||||
}
|
||||
|
||||
func cleanupZopfliCostModel(self *zopfliCostModel) {
|
||||
self.literal_costs_ = nil
|
||||
self.cost_dist_ = nil
|
||||
}
|
||||
|
||||
func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
|
||||
var sum uint = 0
|
||||
var missing_symbol_sum uint
|
||||
var log2sum float32
|
||||
var missing_symbol_cost float32
|
||||
var i uint
|
||||
for i = 0; i < histogram_size; i++ {
|
||||
sum += uint(histogram[i])
|
||||
}
|
||||
|
||||
log2sum = float32(fastLog2(sum))
|
||||
missing_symbol_sum = sum
|
||||
if !literal_histogram {
|
||||
for i = 0; i < histogram_size; i++ {
|
||||
if histogram[i] == 0 {
|
||||
missing_symbol_sum++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
|
||||
for i = 0; i < histogram_size; i++ {
|
||||
if histogram[i] == 0 {
|
||||
cost[i] = missing_symbol_cost
|
||||
continue
|
||||
}
|
||||
|
||||
/* Shannon bits for this symbol. */
|
||||
cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
|
||||
|
||||
/* Cannot be coded with less than 1 bit */
|
||||
if cost[i] < 1 {
|
||||
cost[i] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) {
|
||||
var histogram_literal [numLiteralSymbols]uint32
|
||||
var histogram_cmd [numCommandSymbols]uint32
|
||||
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
|
||||
var cost_literal [numLiteralSymbols]float32
|
||||
var pos uint = position - last_insert_len
|
||||
var min_cost_cmd float32 = kInfinity
|
||||
var cost_cmd []float32 = self.cost_cmd_[:]
|
||||
var literal_costs []float32
|
||||
|
||||
histogram_literal = [numLiteralSymbols]uint32{}
|
||||
histogram_cmd = [numCommandSymbols]uint32{}
|
||||
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
|
||||
|
||||
for i := range commands {
|
||||
var inslength uint = uint(commands[i].insert_len_)
|
||||
var copylength uint = uint(commandCopyLen(&commands[i]))
|
||||
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
|
||||
var cmdcode uint = uint(commands[i].cmd_prefix_)
|
||||
var j uint
|
||||
|
||||
histogram_cmd[cmdcode]++
|
||||
if cmdcode >= 128 {
|
||||
histogram_dist[distcode]++
|
||||
}
|
||||
|
||||
for j = 0; j < inslength; j++ {
|
||||
histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
|
||||
}
|
||||
|
||||
pos += inslength + copylength
|
||||
}
|
||||
|
||||
setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
|
||||
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
|
||||
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
|
||||
|
||||
for i := 0; i < numCommandSymbols; i++ {
|
||||
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
|
||||
}
|
||||
|
||||
self.min_cost_cmd_ = min_cost_cmd
|
||||
{
|
||||
literal_costs = self.literal_costs_
|
||||
var literal_carry float32 = 0.0
|
||||
num_bytes := int(self.num_bytes_)
|
||||
literal_costs[0] = 0.0
|
||||
for i := 0; i < num_bytes; i++ {
|
||||
literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]]
|
||||
literal_costs[i+1] = literal_costs[i] + literal_carry
|
||||
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
||||
var literal_costs []float32 = self.literal_costs_
|
||||
var literal_carry float32 = 0.0
|
||||
var cost_dist []float32 = self.cost_dist_
|
||||
var cost_cmd []float32 = self.cost_cmd_[:]
|
||||
var num_bytes uint = self.num_bytes_
|
||||
var i uint
|
||||
estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
|
||||
literal_costs[0] = 0.0
|
||||
for i = 0; i < num_bytes; i++ {
|
||||
literal_carry += literal_costs[i+1]
|
||||
literal_costs[i+1] = literal_costs[i] + literal_carry
|
||||
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
||||
}
|
||||
|
||||
for i = 0; i < numCommandSymbols; i++ {
|
||||
cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
|
||||
}
|
||||
|
||||
for i = 0; uint32(i) < self.distance_histogram_size; i++ {
|
||||
cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
|
||||
}
|
||||
|
||||
self.min_cost_cmd_ = float32(fastLog2(11))
|
||||
}
|
||||
|
||||
func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
|
||||
return self.cost_cmd_[cmdcode]
|
||||
}
|
||||
|
||||
func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
|
||||
return self.cost_dist_[distcode]
|
||||
}
|
||||
|
||||
func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
|
||||
return self.literal_costs_[to] - self.literal_costs_[from]
|
||||
}
|
||||
|
||||
func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
|
||||
return self.min_cost_cmd_
|
||||
}
|
||||
|
||||
/* REQUIRES: len >= 2, start_pos <= pos */
|
||||
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
|
||||
/* Maintains the "ZopfliNode array invariant". */
|
||||
func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
|
||||
var next *zopfliNode = &nodes[pos+len]
|
||||
next.length = uint32(len | (len+9-len_code)<<25)
|
||||
next.distance = uint32(dist)
|
||||
next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
|
||||
next.u.cost = cost
|
||||
}
|
||||
|
||||
type posData struct {
|
||||
pos uint
|
||||
distance_cache [4]int
|
||||
costdiff float32
|
||||
cost float32
|
||||
}
|
||||
|
||||
/* Maintains the smallest 8 cost difference together with their positions */
|
||||
type startPosQueue struct {
|
||||
q_ [8]posData
|
||||
idx_ uint
|
||||
}
|
||||
|
||||
func initStartPosQueue(self *startPosQueue) {
|
||||
self.idx_ = 0
|
||||
}
|
||||
|
||||
func startPosQueueSize(self *startPosQueue) uint {
|
||||
return brotli_min_size_t(self.idx_, 8)
|
||||
}
|
||||
|
||||
func startPosQueuePush(self *startPosQueue, posdata *posData) {
|
||||
var offset uint = ^(self.idx_) & 7
|
||||
self.idx_++
|
||||
var len uint = startPosQueueSize(self)
|
||||
var i uint
|
||||
var q []posData = self.q_[:]
|
||||
q[offset] = *posdata
|
||||
|
||||
/* Restore the sorted order. In the list of |len| items at most |len - 1|
|
||||
adjacent element comparisons / swaps are required. */
|
||||
for i = 1; i < len; i++ {
|
||||
if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
|
||||
var tmp posData = q[offset&7]
|
||||
q[offset&7] = q[(offset+1)&7]
|
||||
q[(offset+1)&7] = tmp
|
||||
}
|
||||
|
||||
offset++
|
||||
}
|
||||
}
|
||||
|
||||
func startPosQueueAt(self *startPosQueue, k uint) *posData {
|
||||
return &self.q_[(k-self.idx_)&7]
|
||||
}
|
||||
|
||||
/* Returns the minimum possible copy length that can improve the cost of any */
|
||||
/* future position. */
|
||||
func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
|
||||
var min_cost float32 = start_cost
|
||||
var len uint = 2
|
||||
var next_len_bucket uint = 4
|
||||
/* Compute the minimum possible cost of reaching any future position. */
|
||||
|
||||
var next_len_offset uint = 10
|
||||
for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
|
||||
/* We already reached (pos + len) with no more cost than the minimum
|
||||
possible cost of reaching anything from this pos, so there is no point in
|
||||
looking for lengths <= len. */
|
||||
len++
|
||||
|
||||
if len == next_len_offset {
|
||||
/* We reached the next copy length code bucket, so we add one more
|
||||
extra bit to the minimum cost. */
|
||||
min_cost += 1.0
|
||||
|
||||
next_len_offset += next_len_bucket
|
||||
next_len_bucket *= 2
|
||||
}
|
||||
}
|
||||
|
||||
return uint(len)
|
||||
}
|
||||
|
||||
/* REQUIRES: nodes[pos].cost < kInfinity
|
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
||||
func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
|
||||
var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
|
||||
var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
|
||||
var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
|
||||
|
||||
/* Since |block_start + pos| is the end position of the command, the copy part
|
||||
starts from |block_start + pos - clen|. Distances that are greater than
|
||||
this or greater than |max_backward_limit| + |gap| are static dictionary
|
||||
references, and do not update the last distances.
|
||||
Also distance code 0 (last distance) does not update the last distances. */
|
||||
if pos == 0 {
|
||||
return 0
|
||||
} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
|
||||
return uint32(pos)
|
||||
} else {
|
||||
return nodes[pos-clen-ilen].u.shortcut
|
||||
}
|
||||
}
|
||||
|
||||
/* Fills in dist_cache[0..3] with the last four distances (as defined by
|
||||
Section 4. of the Spec) that would be used at (block_start + pos) if we
|
||||
used the shortest path of commands from block_start, computed from
|
||||
nodes[0..pos]. The last four distances at block_start are in
|
||||
starting_dist_cache[0..3].
|
||||
REQUIRES: nodes[pos].cost < kInfinity
|
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
|
||||
func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
|
||||
var idx int = 0
|
||||
var p uint = uint(nodes[pos].u.shortcut)
|
||||
for idx < 4 && p > 0 {
|
||||
var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
|
||||
var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
|
||||
var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
|
||||
dist_cache[idx] = int(dist)
|
||||
idx++
|
||||
|
||||
/* Because of prerequisite, p >= clen + ilen >= 2. */
|
||||
p = uint(nodes[p-clen-ilen].u.shortcut)
|
||||
}
|
||||
|
||||
for ; idx < 4; idx++ {
|
||||
dist_cache[idx] = starting_dist_cache[0]
|
||||
starting_dist_cache = starting_dist_cache[1:]
|
||||
}
|
||||
}
|
||||
|
||||
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
|
||||
is eligible. */
|
||||
func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
|
||||
/* Save cost, because ComputeDistanceCache invalidates it. */
|
||||
var node_cost float32 = nodes[pos].u.cost
|
||||
nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
|
||||
if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
|
||||
var posdata posData
|
||||
posdata.pos = pos
|
||||
posdata.cost = node_cost
|
||||
posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
|
||||
computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
|
||||
startPosQueuePush(queue, &posdata)
|
||||
}
|
||||
}
|
||||
|
||||
/* Returns longest copy length. */
|
||||
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
|
||||
var cur_ix uint = block_start + pos
|
||||
var cur_ix_masked uint = cur_ix & ringbuffer_mask
|
||||
var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
|
||||
var max_len uint = num_bytes - pos
|
||||
var max_zopfli_len uint = maxZopfliLen(params)
|
||||
var max_iters uint = maxZopfliCandidates(params)
|
||||
var min_len uint
|
||||
var result uint = 0
|
||||
var k uint
|
||||
var gap uint = 0
|
||||
|
||||
evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
|
||||
{
|
||||
var posdata *posData = startPosQueueAt(queue, 0)
|
||||
var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
|
||||
min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
|
||||
}
|
||||
|
||||
/* Go over the command starting positions in order of increasing cost
|
||||
difference. */
|
||||
for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
|
||||
var posdata *posData = startPosQueueAt(queue, k)
|
||||
var start uint = posdata.pos
|
||||
var inscode uint16 = getInsertLengthCode(pos - start)
|
||||
var start_costdiff float32 = posdata.costdiff
|
||||
var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
|
||||
var best_len uint = min_len - 1
|
||||
var j uint = 0
|
||||
/* Look for last distance matches using the distance cache from this
|
||||
starting position. */
|
||||
for ; j < numDistanceShortCodes && best_len < max_len; j++ {
|
||||
var idx uint = uint(kDistanceCacheIndex[j])
|
||||
var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
|
||||
var prev_ix uint = cur_ix - backward
|
||||
var len uint = 0
|
||||
var continuation byte = ringbuffer[cur_ix_masked+best_len]
|
||||
if cur_ix_masked+best_len > ringbuffer_mask {
|
||||
break
|
||||
}
|
||||
|
||||
if backward > max_distance+gap {
|
||||
/* Word dictionary -> ignore. */
|
||||
continue
|
||||
}
|
||||
|
||||
if backward <= max_distance {
|
||||
/* Regular backward reference. */
|
||||
if prev_ix >= cur_ix {
|
||||
continue
|
||||
}
|
||||
|
||||
prev_ix &= ringbuffer_mask
|
||||
if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
|
||||
continue
|
||||
}
|
||||
|
||||
len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
|
||||
var l uint
|
||||
for l = best_len + 1; l <= len; l++ {
|
||||
var copycode uint16 = getCopyLengthCode(l)
|
||||
var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
|
||||
var tmp float32
|
||||
if cmdcode < 128 {
|
||||
tmp = base_cost
|
||||
} else {
|
||||
tmp = dist_cost
|
||||
}
|
||||
var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
|
||||
if cost < nodes[pos+l].u.cost {
|
||||
updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
|
||||
result = brotli_max_size_t(result, l)
|
||||
}
|
||||
|
||||
best_len = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* At higher iterations look only for new last distance matches, since
|
||||
looking only for new command start positions with the same distances
|
||||
does not help much. */
|
||||
if k >= 2 {
|
||||
continue
|
||||
}
|
||||
{
|
||||
/* Loop through all possible copy lengths at this position. */
|
||||
var len uint = min_len
|
||||
for j = 0; j < num_matches; j++ {
|
||||
var match backwardMatch = matches[j]
|
||||
var dist uint = uint(match.distance)
|
||||
var is_dictionary_match bool = (dist > max_distance+gap)
|
||||
var dist_code uint = dist + numDistanceShortCodes - 1
|
||||
var dist_symbol uint16
|
||||
var distextra uint32
|
||||
var distnumextra uint32
|
||||
var dist_cost float32
|
||||
var max_match_len uint
|
||||
/* We already tried all possible last distance matches, so we can use
|
||||
normal distance code here. */
|
||||
prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
|
||||
|
||||
distnumextra = uint32(dist_symbol) >> 10
|
||||
dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
|
||||
|
||||
/* Try all copy lengths up until the maximum copy length corresponding
|
||||
to this distance. If the distance refers to the static dictionary, or
|
||||
the maximum length is long enough, try only one maximum length. */
|
||||
max_match_len = backwardMatchLength(&match)
|
||||
|
||||
if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
|
||||
len = max_match_len
|
||||
}
|
||||
|
||||
for ; len <= max_match_len; len++ {
|
||||
var len_code uint
|
||||
if is_dictionary_match {
|
||||
len_code = backwardMatchLengthCode(&match)
|
||||
} else {
|
||||
len_code = len
|
||||
}
|
||||
var copycode uint16 = getCopyLengthCode(len_code)
|
||||
var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
|
||||
var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
|
||||
if cost < nodes[pos+len].u.cost {
|
||||
updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
|
||||
if len > result {
|
||||
result = len
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
|
||||
var index uint = num_bytes
|
||||
var num_commands uint = 0
|
||||
for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
|
||||
index--
|
||||
}
|
||||
nodes[index].u.next = math.MaxUint32
|
||||
for index != 0 {
|
||||
var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
|
||||
index -= uint(len)
|
||||
nodes[index].u.next = uint32(len)
|
||||
num_commands++
|
||||
}
|
||||
|
||||
return num_commands
|
||||
}
|
||||
|
||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
||||
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) {
|
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||
var pos uint = 0
|
||||
var offset uint32 = nodes[0].u.next
|
||||
var i uint
|
||||
var gap uint = 0
|
||||
for i = 0; offset != math.MaxUint32; i++ {
|
||||
var next *zopfliNode = &nodes[uint32(pos)+offset]
|
||||
var copy_length uint = uint(zopfliNodeCopyLength(next))
|
||||
var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
|
||||
pos += insert_length
|
||||
offset = next.u.next
|
||||
if i == 0 {
|
||||
insert_length += *last_insert_len
|
||||
*last_insert_len = 0
|
||||
}
|
||||
{
|
||||
var distance uint = uint(zopfliNodeCopyDistance(next))
|
||||
var len_code uint = uint(zopfliNodeLengthCode(next))
|
||||
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
|
||||
var is_dictionary bool = (distance > max_distance+gap)
|
||||
var dist_code uint = uint(zopfliNodeDistanceCode(next))
|
||||
*commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code))
|
||||
|
||||
if !is_dictionary && dist_code > 0 {
|
||||
dist_cache[3] = dist_cache[2]
|
||||
dist_cache[2] = dist_cache[1]
|
||||
dist_cache[1] = dist_cache[0]
|
||||
dist_cache[0] = int(distance)
|
||||
}
|
||||
}
|
||||
|
||||
*num_literals += insert_length
|
||||
pos += copy_length
|
||||
}
|
||||
|
||||
*last_insert_len += num_bytes - pos
|
||||
}
|
||||
|
||||
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
|
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||
var max_zopfli_len uint = maxZopfliLen(params)
|
||||
var queue startPosQueue
|
||||
var cur_match_pos uint = 0
|
||||
var i uint
|
||||
nodes[0].length = 0
|
||||
nodes[0].u.cost = 0
|
||||
initStartPosQueue(&queue)
|
||||
for i = 0; i+3 < num_bytes; i++ {
|
||||
var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
|
||||
if skip < longCopyQuickStep {
|
||||
skip = 0
|
||||
}
|
||||
cur_match_pos += uint(num_matches[i])
|
||||
if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
|
||||
skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
|
||||
}
|
||||
|
||||
if skip > 1 {
|
||||
skip--
|
||||
for skip != 0 {
|
||||
i++
|
||||
if i+3 >= num_bytes {
|
||||
break
|
||||
}
|
||||
evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
|
||||
cur_match_pos += uint(num_matches[i])
|
||||
skip--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return computeShortestPathFromNodes(num_bytes, nodes)
|
||||
}
|
||||
|
||||
/* Computes the shortest path of commands from position to at most
|
||||
position + num_bytes.
|
||||
|
||||
On return, path->size() is the number of commands found and path[i] is the
|
||||
length of the i-th command (copy length plus insert length).
|
||||
Note that the sum of the lengths of all commands can be less than num_bytes.
|
||||
|
||||
On return, the nodes[0..num_bytes] array will have the following
|
||||
"ZopfliNode array invariant":
|
||||
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
|
||||
(1) nodes[i].copy_length() >= 2
|
||||
(2) nodes[i].command_length() <= i and
|
||||
(3) nodes[i - nodes[i].command_length()].cost < kInfinity
|
||||
|
||||
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
|
||||
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
|
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||
var max_zopfli_len uint = maxZopfliLen(params)
|
||||
var model zopfliCostModel
|
||||
var queue startPosQueue
|
||||
var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
|
||||
var store_end uint
|
||||
if num_bytes >= hasher.StoreLookahead() {
|
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1
|
||||
} else {
|
||||
store_end = position
|
||||
}
|
||||
var i uint
|
||||
var gap uint = 0
|
||||
var lz_matches_offset uint = 0
|
||||
nodes[0].length = 0
|
||||
nodes[0].u.cost = 0
|
||||
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
||||
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
||||
initStartPosQueue(&queue)
|
||||
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
|
||||
var pos uint = position + i
|
||||
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
|
||||
var skip uint
|
||||
var num_matches uint
|
||||
num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
|
||||
if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
|
||||
matches[0] = matches[num_matches-1]
|
||||
num_matches = 1
|
||||
}
|
||||
|
||||
skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
|
||||
if skip < longCopyQuickStep {
|
||||
skip = 0
|
||||
}
|
||||
if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
|
||||
skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
|
||||
}
|
||||
|
||||
if skip > 1 {
|
||||
/* Add the tail of the copy to the hasher. */
|
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
|
||||
|
||||
skip--
|
||||
for skip != 0 {
|
||||
i++
|
||||
if i+hasher.HashTypeLength()-1 >= num_bytes {
|
||||
break
|
||||
}
|
||||
evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
|
||||
skip--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanupZopfliCostModel(&model)
|
||||
return computeShortestPathFromNodes(num_bytes, nodes)
|
||||
}
|
||||
|
||||
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||
var nodes []zopfliNode
|
||||
nodes = make([]zopfliNode, (num_bytes + 1))
|
||||
initZopfliNodes(nodes, num_bytes+1)
|
||||
zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
|
||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
||||
nodes = nil
|
||||
}
|
||||
|
||||
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||
var num_matches []uint32 = make([]uint32, num_bytes)
|
||||
var matches_size uint = 4 * num_bytes
|
||||
var store_end uint
|
||||
if num_bytes >= hasher.StoreLookahead() {
|
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1
|
||||
} else {
|
||||
store_end = position
|
||||
}
|
||||
var cur_match_pos uint = 0
|
||||
var i uint
|
||||
var orig_num_literals uint
|
||||
var orig_last_insert_len uint
|
||||
var orig_dist_cache [4]int
|
||||
var orig_num_commands int
|
||||
var model zopfliCostModel
|
||||
var nodes []zopfliNode
|
||||
var matches []backwardMatch = make([]backwardMatch, matches_size)
|
||||
var gap uint = 0
|
||||
var shadow_matches uint = 0
|
||||
var new_array []backwardMatch
|
||||
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
|
||||
var pos uint = position + i
|
||||
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
|
||||
var max_length uint = num_bytes - i
|
||||
var num_found_matches uint
|
||||
var cur_match_end uint
|
||||
var j uint
|
||||
|
||||
/* Ensure that we have enough free slots. */
|
||||
if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
|
||||
var new_size uint = matches_size
|
||||
if new_size == 0 {
|
||||
new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
|
||||
}
|
||||
|
||||
for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
|
||||
new_size *= 2
|
||||
}
|
||||
|
||||
new_array = make([]backwardMatch, new_size)
|
||||
if matches_size != 0 {
|
||||
copy(new_array, matches[:matches_size])
|
||||
}
|
||||
|
||||
matches = new_array
|
||||
matches_size = new_size
|
||||
}
|
||||
|
||||
num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
|
||||
cur_match_end = cur_match_pos + num_found_matches
|
||||
for j = cur_match_pos; j+1 < cur_match_end; j++ {
|
||||
assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
|
||||
}
|
||||
|
||||
num_matches[i] = uint32(num_found_matches)
|
||||
if num_found_matches > 0 {
|
||||
var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
|
||||
if match_len > maxZopfliLenQuality11 {
|
||||
var skip uint = match_len - 1
|
||||
matches[cur_match_pos] = matches[cur_match_end-1]
|
||||
cur_match_pos++
|
||||
num_matches[i] = 1
|
||||
|
||||
/* Add the tail of the copy to the hasher. */
|
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
|
||||
var pos uint = i
|
||||
for i := 0; i < int(skip); i++ {
|
||||
num_matches[pos+1:][i] = 0
|
||||
}
|
||||
i += skip
|
||||
} else {
|
||||
cur_match_pos = cur_match_end
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
orig_num_literals = *num_literals
|
||||
orig_last_insert_len = *last_insert_len
|
||||
copy(orig_dist_cache[:], dist_cache[:4])
|
||||
orig_num_commands = len(*commands)
|
||||
nodes = make([]zopfliNode, (num_bytes + 1))
|
||||
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
||||
for i = 0; i < 2; i++ {
|
||||
initZopfliNodes(nodes, num_bytes+1)
|
||||
if i == 0 {
|
||||
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
||||
} else {
|
||||
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len)
|
||||
}
|
||||
|
||||
*commands = (*commands)[:orig_num_commands]
|
||||
*num_literals = orig_num_literals
|
||||
*last_insert_len = orig_last_insert_len
|
||||
copy(dist_cache, orig_dist_cache[:4])
|
||||
zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
|
||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
||||
}
|
||||
|
||||
cleanupZopfliCostModel(&model)
|
||||
nodes = nil
|
||||
matches = nil
|
||||
num_matches = nil
|
||||
}
|
||||
436
vendor/github.com/andybalholm/brotli/bit_cost.go
generated
vendored
Normal file
436
vendor/github.com/andybalholm/brotli/bit_cost.go
generated
vendored
Normal file
@ -0,0 +1,436 @@
|
||||
package brotli
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions to estimate the bit cost of Huffman trees. */
|
||||
func shannonEntropy(population []uint32, size uint, total *uint) float64 {
|
||||
var sum uint = 0
|
||||
var retval float64 = 0
|
||||
var population_end []uint32 = population[size:]
|
||||
var p uint
|
||||
for -cap(population) < -cap(population_end) {
|
||||
p = uint(population[0])
|
||||
population = population[1:]
|
||||
sum += p
|
||||
retval -= float64(p) * fastLog2(p)
|
||||
}
|
||||
|
||||
if sum != 0 {
|
||||
retval += float64(sum) * fastLog2(sum)
|
||||
}
|
||||
*total = sum
|
||||
return retval
|
||||
}
|
||||
|
||||
func bitsEntropy(population []uint32, size uint) float64 {
|
||||
var sum uint
|
||||
var retval float64 = shannonEntropy(population, size, &sum)
|
||||
if retval < float64(sum) {
|
||||
/* At least one bit per literal is needed. */
|
||||
retval = float64(sum)
|
||||
}
|
||||
|
||||
return retval
|
||||
}
|
||||
|
||||
const kOneSymbolHistogramCost float64 = 12
|
||||
const kTwoSymbolHistogramCost float64 = 20
|
||||
const kThreeSymbolHistogramCost float64 = 28
|
||||
const kFourSymbolHistogramCost float64 = 37
|
||||
|
||||
func populationCostLiteral(histogram *histogramLiteral) float64 {
|
||||
var data_size uint = histogramDataSizeLiteral()
|
||||
var count int = 0
|
||||
var s [5]uint
|
||||
var bits float64 = 0.0
|
||||
var i uint
|
||||
if histogram.total_count_ == 0 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
for i = 0; i < data_size; i++ {
|
||||
if histogram.data_[i] > 0 {
|
||||
s[count] = i
|
||||
count++
|
||||
if count > 4 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
if count == 2 {
|
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
||||
}
|
||||
|
||||
if count == 3 {
|
||||
var histo0 uint32 = histogram.data_[s[0]]
|
||||
var histo1 uint32 = histogram.data_[s[1]]
|
||||
var histo2 uint32 = histogram.data_[s[2]]
|
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
||||
}
|
||||
|
||||
if count == 4 {
|
||||
var histo [4]uint32
|
||||
var h23 uint32
|
||||
var histomax uint32
|
||||
for i = 0; i < 4; i++ {
|
||||
histo[i] = histogram.data_[s[i]]
|
||||
}
|
||||
|
||||
/* Sort */
|
||||
for i = 0; i < 4; i++ {
|
||||
var j uint
|
||||
for j = i + 1; j < 4; j++ {
|
||||
if histo[j] > histo[i] {
|
||||
var tmp uint32 = histo[j]
|
||||
histo[j] = histo[i]
|
||||
histo[i] = tmp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
h23 = histo[2] + histo[3]
|
||||
histomax = brotli_max_uint32_t(h23, histo[0])
|
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
||||
}
|
||||
{
|
||||
var max_depth uint = 1
|
||||
var depth_histo = [codeLengthCodes]uint32{0}
|
||||
/* In this loop we compute the entropy of the histogram and simultaneously
|
||||
build a simplified histogram of the code length codes where we use the
|
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_)
|
||||
for i = 0; i < data_size; {
|
||||
if histogram.data_[i] > 0 {
|
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
||||
= log2(total_count) - log2(count(symbol)) */
|
||||
|
||||
var depth uint = uint(log2p + 0.5)
|
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
||||
bits += float64(histogram.data_[i]) * log2p
|
||||
|
||||
if depth > 15 {
|
||||
depth = 15
|
||||
}
|
||||
|
||||
if depth > max_depth {
|
||||
max_depth = depth
|
||||
}
|
||||
|
||||
depth_histo[depth]++
|
||||
i++
|
||||
} else {
|
||||
var reps uint32 = 1
|
||||
/* Compute the run length of zeros and add the appropriate number of 0
|
||||
and 17 code length codes to the code length code histogram. */
|
||||
|
||||
var k uint
|
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
||||
reps++
|
||||
}
|
||||
|
||||
i += uint(reps)
|
||||
if i == data_size {
|
||||
/* Don't add any cost for the last zero run, since these are encoded
|
||||
only implicitly. */
|
||||
break
|
||||
}
|
||||
|
||||
if reps < 3 {
|
||||
depth_histo[0] += reps
|
||||
} else {
|
||||
reps -= 2
|
||||
for reps > 0 {
|
||||
depth_histo[repeatZeroCodeLength]++
|
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */
|
||||
bits += 3
|
||||
|
||||
reps >>= 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */
|
||||
bits += float64(18 + 2*max_depth)
|
||||
|
||||
/* Add the entropy of the code length code histogram. */
|
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
||||
}
|
||||
|
||||
return bits
|
||||
}
|
||||
|
||||
func populationCostCommand(histogram *histogramCommand) float64 {
|
||||
var data_size uint = histogramDataSizeCommand()
|
||||
var count int = 0
|
||||
var s [5]uint
|
||||
var bits float64 = 0.0
|
||||
var i uint
|
||||
if histogram.total_count_ == 0 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
for i = 0; i < data_size; i++ {
|
||||
if histogram.data_[i] > 0 {
|
||||
s[count] = i
|
||||
count++
|
||||
if count > 4 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
if count == 2 {
|
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
||||
}
|
||||
|
||||
if count == 3 {
|
||||
var histo0 uint32 = histogram.data_[s[0]]
|
||||
var histo1 uint32 = histogram.data_[s[1]]
|
||||
var histo2 uint32 = histogram.data_[s[2]]
|
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
||||
}
|
||||
|
||||
if count == 4 {
|
||||
var histo [4]uint32
|
||||
var h23 uint32
|
||||
var histomax uint32
|
||||
for i = 0; i < 4; i++ {
|
||||
histo[i] = histogram.data_[s[i]]
|
||||
}
|
||||
|
||||
/* Sort */
|
||||
for i = 0; i < 4; i++ {
|
||||
var j uint
|
||||
for j = i + 1; j < 4; j++ {
|
||||
if histo[j] > histo[i] {
|
||||
var tmp uint32 = histo[j]
|
||||
histo[j] = histo[i]
|
||||
histo[i] = tmp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
h23 = histo[2] + histo[3]
|
||||
histomax = brotli_max_uint32_t(h23, histo[0])
|
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
||||
}
|
||||
{
|
||||
var max_depth uint = 1
|
||||
var depth_histo = [codeLengthCodes]uint32{0}
|
||||
/* In this loop we compute the entropy of the histogram and simultaneously
|
||||
build a simplified histogram of the code length codes where we use the
|
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_)
|
||||
for i = 0; i < data_size; {
|
||||
if histogram.data_[i] > 0 {
|
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
||||
= log2(total_count) - log2(count(symbol)) */
|
||||
|
||||
var depth uint = uint(log2p + 0.5)
|
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
||||
bits += float64(histogram.data_[i]) * log2p
|
||||
|
||||
if depth > 15 {
|
||||
depth = 15
|
||||
}
|
||||
|
||||
if depth > max_depth {
|
||||
max_depth = depth
|
||||
}
|
||||
|
||||
depth_histo[depth]++
|
||||
i++
|
||||
} else {
|
||||
var reps uint32 = 1
|
||||
/* Compute the run length of zeros and add the appropriate number of 0
|
||||
and 17 code length codes to the code length code histogram. */
|
||||
|
||||
var k uint
|
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
||||
reps++
|
||||
}
|
||||
|
||||
i += uint(reps)
|
||||
if i == data_size {
|
||||
/* Don't add any cost for the last zero run, since these are encoded
|
||||
only implicitly. */
|
||||
break
|
||||
}
|
||||
|
||||
if reps < 3 {
|
||||
depth_histo[0] += reps
|
||||
} else {
|
||||
reps -= 2
|
||||
for reps > 0 {
|
||||
depth_histo[repeatZeroCodeLength]++
|
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */
|
||||
bits += 3
|
||||
|
||||
reps >>= 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */
|
||||
bits += float64(18 + 2*max_depth)
|
||||
|
||||
/* Add the entropy of the code length code histogram. */
|
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
||||
}
|
||||
|
||||
return bits
|
||||
}
|
||||
|
||||
func populationCostDistance(histogram *histogramDistance) float64 {
|
||||
var data_size uint = histogramDataSizeDistance()
|
||||
var count int = 0
|
||||
var s [5]uint
|
||||
var bits float64 = 0.0
|
||||
var i uint
|
||||
if histogram.total_count_ == 0 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
for i = 0; i < data_size; i++ {
|
||||
if histogram.data_[i] > 0 {
|
||||
s[count] = i
|
||||
count++
|
||||
if count > 4 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return kOneSymbolHistogramCost
|
||||
}
|
||||
|
||||
if count == 2 {
|
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
|
||||
}
|
||||
|
||||
if count == 3 {
|
||||
var histo0 uint32 = histogram.data_[s[0]]
|
||||
var histo1 uint32 = histogram.data_[s[1]]
|
||||
var histo2 uint32 = histogram.data_[s[2]]
|
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
|
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
|
||||
}
|
||||
|
||||
if count == 4 {
|
||||
var histo [4]uint32
|
||||
var h23 uint32
|
||||
var histomax uint32
|
||||
for i = 0; i < 4; i++ {
|
||||
histo[i] = histogram.data_[s[i]]
|
||||
}
|
||||
|
||||
/* Sort */
|
||||
for i = 0; i < 4; i++ {
|
||||
var j uint
|
||||
for j = i + 1; j < 4; j++ {
|
||||
if histo[j] > histo[i] {
|
||||
var tmp uint32 = histo[j]
|
||||
histo[j] = histo[i]
|
||||
histo[i] = tmp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
h23 = histo[2] + histo[3]
|
||||
histomax = brotli_max_uint32_t(h23, histo[0])
|
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
|
||||
}
|
||||
{
|
||||
var max_depth uint = 1
|
||||
var depth_histo = [codeLengthCodes]uint32{0}
|
||||
/* In this loop we compute the entropy of the histogram and simultaneously
|
||||
build a simplified histogram of the code length codes where we use the
|
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */
|
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_)
|
||||
for i = 0; i < data_size; {
|
||||
if histogram.data_[i] > 0 {
|
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
|
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
|
||||
= log2(total_count) - log2(count(symbol)) */
|
||||
|
||||
var depth uint = uint(log2p + 0.5)
|
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */
|
||||
bits += float64(histogram.data_[i]) * log2p
|
||||
|
||||
if depth > 15 {
|
||||
depth = 15
|
||||
}
|
||||
|
||||
if depth > max_depth {
|
||||
max_depth = depth
|
||||
}
|
||||
|
||||
depth_histo[depth]++
|
||||
i++
|
||||
} else {
|
||||
var reps uint32 = 1
|
||||
/* Compute the run length of zeros and add the appropriate number of 0
|
||||
and 17 code length codes to the code length code histogram. */
|
||||
|
||||
var k uint
|
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
|
||||
reps++
|
||||
}
|
||||
|
||||
i += uint(reps)
|
||||
if i == data_size {
|
||||
/* Don't add any cost for the last zero run, since these are encoded
|
||||
only implicitly. */
|
||||
break
|
||||
}
|
||||
|
||||
if reps < 3 {
|
||||
depth_histo[0] += reps
|
||||
} else {
|
||||
reps -= 2
|
||||
for reps > 0 {
|
||||
depth_histo[repeatZeroCodeLength]++
|
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */
|
||||
bits += 3
|
||||
|
||||
reps >>= 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */
|
||||
bits += float64(18 + 2*max_depth)
|
||||
|
||||
/* Add the entropy of the code length code histogram. */
|
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
|
||||
}
|
||||
|
||||
return bits
|
||||
}
|
||||
266
vendor/github.com/andybalholm/brotli/bit_reader.go
generated
vendored
Normal file
266
vendor/github.com/andybalholm/brotli/bit_reader.go
generated
vendored
Normal file
@ -0,0 +1,266 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Bit reading helpers */
|
||||
|
||||
const shortFillBitWindowRead = (8 >> 1)
|
||||
|
||||
var kBitMask = [33]uint32{
|
||||
0x00000000,
|
||||
0x00000001,
|
||||
0x00000003,
|
||||
0x00000007,
|
||||
0x0000000F,
|
||||
0x0000001F,
|
||||
0x0000003F,
|
||||
0x0000007F,
|
||||
0x000000FF,
|
||||
0x000001FF,
|
||||
0x000003FF,
|
||||
0x000007FF,
|
||||
0x00000FFF,
|
||||
0x00001FFF,
|
||||
0x00003FFF,
|
||||
0x00007FFF,
|
||||
0x0000FFFF,
|
||||
0x0001FFFF,
|
||||
0x0003FFFF,
|
||||
0x0007FFFF,
|
||||
0x000FFFFF,
|
||||
0x001FFFFF,
|
||||
0x003FFFFF,
|
||||
0x007FFFFF,
|
||||
0x00FFFFFF,
|
||||
0x01FFFFFF,
|
||||
0x03FFFFFF,
|
||||
0x07FFFFFF,
|
||||
0x0FFFFFFF,
|
||||
0x1FFFFFFF,
|
||||
0x3FFFFFFF,
|
||||
0x7FFFFFFF,
|
||||
0xFFFFFFFF,
|
||||
}
|
||||
|
||||
func bitMask(n uint32) uint32 {
|
||||
return kBitMask[n]
|
||||
}
|
||||
|
||||
type bitReader struct {
|
||||
val_ uint64
|
||||
bit_pos_ uint32
|
||||
input []byte
|
||||
input_len uint
|
||||
byte_pos uint
|
||||
}
|
||||
|
||||
type bitReaderState struct {
|
||||
val_ uint64
|
||||
bit_pos_ uint32
|
||||
input []byte
|
||||
input_len uint
|
||||
byte_pos uint
|
||||
}
|
||||
|
||||
/* Initializes the BrotliBitReader fields. */
|
||||
|
||||
/* Ensures that accumulator is not empty.
|
||||
May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
|
||||
Returns false if data is required but there is no input available.
|
||||
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
|
||||
reading. */
|
||||
func bitReaderSaveState(from *bitReader, to *bitReaderState) {
|
||||
to.val_ = from.val_
|
||||
to.bit_pos_ = from.bit_pos_
|
||||
to.input = from.input
|
||||
to.input_len = from.input_len
|
||||
to.byte_pos = from.byte_pos
|
||||
}
|
||||
|
||||
func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
|
||||
to.val_ = from.val_
|
||||
to.bit_pos_ = from.bit_pos_
|
||||
to.input = from.input
|
||||
to.input_len = from.input_len
|
||||
to.byte_pos = from.byte_pos
|
||||
}
|
||||
|
||||
func getAvailableBits(br *bitReader) uint32 {
|
||||
return 64 - br.bit_pos_
|
||||
}
|
||||
|
||||
/* Returns amount of unread bytes the bit reader still has buffered from the
|
||||
BrotliInput, including whole bytes in br->val_. */
|
||||
func getRemainingBytes(br *bitReader) uint {
|
||||
return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
|
||||
}
|
||||
|
||||
/* Checks if there is at least |num| bytes left in the input ring-buffer
|
||||
(excluding the bits remaining in br->val_). */
|
||||
func checkInputAmount(br *bitReader, num uint) bool {
|
||||
return br.input_len-br.byte_pos >= num
|
||||
}
|
||||
|
||||
/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
|
||||
Precondition: accumulator contains at least 1 bit.
|
||||
|n_bits| should be in the range [1..24] for regular build. For portable
|
||||
non-64-bit little-endian build only 16 bits are safe to request. */
|
||||
func fillBitWindow(br *bitReader, n_bits uint32) {
|
||||
if br.bit_pos_ >= 32 {
|
||||
br.val_ >>= 32
|
||||
br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
|
||||
br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
|
||||
br.byte_pos += 4
|
||||
}
|
||||
}
|
||||
|
||||
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
|
||||
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
|
||||
func fillBitWindow16(br *bitReader) {
|
||||
fillBitWindow(br, 17)
|
||||
}
|
||||
|
||||
/* Tries to pull one byte of input to accumulator.
|
||||
Returns false if there is no input available. */
|
||||
func pullByte(br *bitReader) bool {
|
||||
if br.byte_pos == br.input_len {
|
||||
return false
|
||||
}
|
||||
|
||||
br.val_ >>= 8
|
||||
br.val_ |= (uint64(br.input[br.byte_pos])) << 56
|
||||
br.bit_pos_ -= 8
|
||||
br.byte_pos++
|
||||
return true
|
||||
}
|
||||
|
||||
/* Returns currently available bits.
|
||||
The number of valid bits could be calculated by BrotliGetAvailableBits. */
|
||||
func getBitsUnmasked(br *bitReader) uint64 {
|
||||
return br.val_ >> br.bit_pos_
|
||||
}
|
||||
|
||||
/* Like BrotliGetBits, but does not mask the result.
|
||||
The result contains at least 16 valid bits. */
|
||||
func get16BitsUnmasked(br *bitReader) uint32 {
|
||||
fillBitWindow(br, 16)
|
||||
return uint32(getBitsUnmasked(br))
|
||||
}
|
||||
|
||||
/* Returns the specified number of bits from |br| without advancing bit
|
||||
position. */
|
||||
func getBits(br *bitReader, n_bits uint32) uint32 {
|
||||
fillBitWindow(br, n_bits)
|
||||
return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
||||
}
|
||||
|
||||
/* Tries to peek the specified amount of bits. Returns false, if there
|
||||
is not enough input. */
|
||||
func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
|
||||
for getAvailableBits(br) < n_bits {
|
||||
if !pullByte(br) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
||||
return true
|
||||
}
|
||||
|
||||
/* Advances the bit pos by |n_bits|. */
|
||||
func dropBits(br *bitReader, n_bits uint32) {
|
||||
br.bit_pos_ += n_bits
|
||||
}
|
||||
|
||||
func bitReaderUnload(br *bitReader) {
|
||||
var unused_bytes uint32 = getAvailableBits(br) >> 3
|
||||
var unused_bits uint32 = unused_bytes << 3
|
||||
br.byte_pos -= uint(unused_bytes)
|
||||
if unused_bits == 64 {
|
||||
br.val_ = 0
|
||||
} else {
|
||||
br.val_ <<= unused_bits
|
||||
}
|
||||
|
||||
br.bit_pos_ += unused_bits
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Precondition: accumulator MUST contain at least |n_bits|. */
|
||||
func takeBits(br *bitReader, n_bits uint32, val *uint32) {
|
||||
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
|
||||
dropBits(br, n_bits)
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Assumes that there is enough input to perform BrotliFillBitWindow. */
|
||||
func readBits(br *bitReader, n_bits uint32) uint32 {
|
||||
var val uint32
|
||||
fillBitWindow(br, n_bits)
|
||||
takeBits(br, n_bits, &val)
|
||||
return val
|
||||
}
|
||||
|
||||
/* Tries to read the specified amount of bits. Returns false, if there
|
||||
is not enough input. |n_bits| MUST be positive. */
|
||||
func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
|
||||
for getAvailableBits(br) < n_bits {
|
||||
if !pullByte(br) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
takeBits(br, n_bits, val)
|
||||
return true
|
||||
}
|
||||
|
||||
/* Advances the bit reader position to the next byte boundary and verifies
|
||||
that any skipped bits are set to zero. */
|
||||
func bitReaderJumpToByteBoundary(br *bitReader) bool {
|
||||
var pad_bits_count uint32 = getAvailableBits(br) & 0x7
|
||||
var pad_bits uint32 = 0
|
||||
if pad_bits_count != 0 {
|
||||
takeBits(br, pad_bits_count, &pad_bits)
|
||||
}
|
||||
|
||||
return pad_bits == 0
|
||||
}
|
||||
|
||||
/* Copies remaining input bytes stored in the bit reader to the output. Value
|
||||
|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
|
||||
warmed up again after this. */
|
||||
func copyBytes(dest []byte, br *bitReader, num uint) {
|
||||
for getAvailableBits(br) >= 8 && num > 0 {
|
||||
dest[0] = byte(getBitsUnmasked(br))
|
||||
dropBits(br, 8)
|
||||
dest = dest[1:]
|
||||
num--
|
||||
}
|
||||
|
||||
copy(dest, br.input[br.byte_pos:][:num])
|
||||
br.byte_pos += num
|
||||
}
|
||||
|
||||
func initBitReader(br *bitReader) {
|
||||
br.val_ = 0
|
||||
br.bit_pos_ = 64
|
||||
}
|
||||
|
||||
func warmupBitReader(br *bitReader) bool {
|
||||
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
|
||||
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
|
||||
enough space in accumulator to fix alignment. */
|
||||
if getAvailableBits(br) == 0 {
|
||||
if !pullByte(br) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
144
vendor/github.com/andybalholm/brotli/block_splitter.go
generated
vendored
Normal file
144
vendor/github.com/andybalholm/brotli/block_splitter.go
generated
vendored
Normal file
@ -0,0 +1,144 @@
|
||||
package brotli
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Block split point selection utilities. */
|
||||
|
||||
type blockSplit struct {
|
||||
num_types uint
|
||||
num_blocks uint
|
||||
types []byte
|
||||
lengths []uint32
|
||||
types_alloc_size uint
|
||||
lengths_alloc_size uint
|
||||
}
|
||||
|
||||
const (
|
||||
kMaxLiteralHistograms uint = 100
|
||||
kMaxCommandHistograms uint = 50
|
||||
kLiteralBlockSwitchCost float64 = 28.1
|
||||
kCommandBlockSwitchCost float64 = 13.5
|
||||
kDistanceBlockSwitchCost float64 = 14.6
|
||||
kLiteralStrideLength uint = 70
|
||||
kCommandStrideLength uint = 40
|
||||
kSymbolsPerLiteralHistogram uint = 544
|
||||
kSymbolsPerCommandHistogram uint = 530
|
||||
kSymbolsPerDistanceHistogram uint = 544
|
||||
kMinLengthForBlockSplitting uint = 128
|
||||
kIterMulForRefining uint = 2
|
||||
kMinItersForRefining uint = 100
|
||||
)
|
||||
|
||||
func countLiterals(cmds []command) uint {
|
||||
var total_length uint = 0
|
||||
/* Count how many we have. */
|
||||
|
||||
for i := range cmds {
|
||||
total_length += uint(cmds[i].insert_len_)
|
||||
}
|
||||
|
||||
return total_length
|
||||
}
|
||||
|
||||
func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) {
|
||||
var pos uint = 0
|
||||
var from_pos uint = offset & mask
|
||||
for i := range cmds {
|
||||
var insert_len uint = uint(cmds[i].insert_len_)
|
||||
if from_pos+insert_len > mask {
|
||||
var head_size uint = mask + 1 - from_pos
|
||||
copy(literals[pos:], data[from_pos:][:head_size])
|
||||
from_pos = 0
|
||||
pos += head_size
|
||||
insert_len -= head_size
|
||||
}
|
||||
|
||||
if insert_len > 0 {
|
||||
copy(literals[pos:], data[from_pos:][:insert_len])
|
||||
pos += insert_len
|
||||
}
|
||||
|
||||
from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
|
||||
}
|
||||
}
|
||||
|
||||
func myRand(seed *uint32) uint32 {
|
||||
/* Initial seed should be 7. In this case, loop length is (1 << 29). */
|
||||
*seed *= 16807
|
||||
|
||||
return *seed
|
||||
}
|
||||
|
||||
func bitCost(count uint) float64 {
|
||||
if count == 0 {
|
||||
return -2.0
|
||||
} else {
|
||||
return fastLog2(count)
|
||||
}
|
||||
}
|
||||
|
||||
const histogramsPerBatch = 64
|
||||
|
||||
const clustersPerBatch = 16
|
||||
|
||||
func initBlockSplit(self *blockSplit) {
|
||||
self.num_types = 0
|
||||
self.num_blocks = 0
|
||||
self.types = self.types[:0]
|
||||
self.lengths = self.lengths[:0]
|
||||
self.types_alloc_size = 0
|
||||
self.lengths_alloc_size = 0
|
||||
}
|
||||
|
||||
func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
|
||||
{
|
||||
var literals_count uint = countLiterals(cmds)
|
||||
var literals []byte = make([]byte, literals_count)
|
||||
|
||||
/* Create a continuous array of literals. */
|
||||
copyLiteralsToByteArray(cmds, data, pos, mask, literals)
|
||||
|
||||
/* Create the block split on the array of literals.
|
||||
Literal histograms have alphabet size 256. */
|
||||
splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
|
||||
|
||||
literals = nil
|
||||
}
|
||||
{
|
||||
var insert_and_copy_codes []uint16 = make([]uint16, len(cmds))
|
||||
/* Compute prefix codes for commands. */
|
||||
|
||||
for i := range cmds {
|
||||
insert_and_copy_codes[i] = cmds[i].cmd_prefix_
|
||||
}
|
||||
|
||||
/* Create the block split on the array of command prefixes. */
|
||||
splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
|
||||
|
||||
/* TODO: reuse for distances? */
|
||||
|
||||
insert_and_copy_codes = nil
|
||||
}
|
||||
{
|
||||
var distance_prefixes []uint16 = make([]uint16, len(cmds))
|
||||
var j uint = 0
|
||||
/* Create a continuous array of distance prefixes. */
|
||||
|
||||
for i := range cmds {
|
||||
var cmd *command = &cmds[i]
|
||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
||||
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
|
||||
j++
|
||||
}
|
||||
}
|
||||
|
||||
/* Create the block split on the array of distance prefixes. */
|
||||
splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
|
||||
|
||||
distance_prefixes = nil
|
||||
}
|
||||
}
|
||||
434
vendor/github.com/andybalholm/brotli/block_splitter_command.go
generated
vendored
Normal file
434
vendor/github.com/andybalholm/brotli/block_splitter_command.go
generated
vendored
Normal file
@ -0,0 +1,434 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
|
||||
var seed uint32 = 7
|
||||
var block_length uint = length / num_histograms
|
||||
var i uint
|
||||
clearHistogramsCommand(histograms, num_histograms)
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
var pos uint = length * i / num_histograms
|
||||
if i != 0 {
|
||||
pos += uint(myRand(&seed) % uint32(block_length))
|
||||
}
|
||||
|
||||
if pos+stride >= length {
|
||||
pos = length - stride - 1
|
||||
}
|
||||
|
||||
histogramAddVectorCommand(&histograms[i], data[pos:], stride)
|
||||
}
|
||||
}
|
||||
|
||||
func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
|
||||
var pos uint = 0
|
||||
if stride >= length {
|
||||
stride = length
|
||||
} else {
|
||||
pos = uint(myRand(seed) % uint32(length-stride+1))
|
||||
}
|
||||
|
||||
histogramAddVectorCommand(sample, data[pos:], stride)
|
||||
}
|
||||
|
||||
func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
|
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
|
||||
var seed uint32 = 7
|
||||
var iter uint
|
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
|
||||
for iter = 0; iter < iters; iter++ {
|
||||
var sample histogramCommand
|
||||
histogramClearCommand(&sample)
|
||||
randomSampleCommand(&seed, data, length, stride, &sample)
|
||||
histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
|
||||
}
|
||||
}
|
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element
|
||||
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
||||
Returns the number of blocks, i.e. one plus the number of block switches. */
|
||||
func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
|
||||
var data_size uint = histogramDataSizeCommand()
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var num_blocks uint = 1
|
||||
var i uint
|
||||
var j uint
|
||||
assert(num_histograms <= 256)
|
||||
if num_histograms <= 1 {
|
||||
for i = 0; i < length; i++ {
|
||||
block_id[i] = 0
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ {
|
||||
insert_cost[i] = 0
|
||||
}
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
|
||||
}
|
||||
|
||||
for i = data_size; i != 0; {
|
||||
i--
|
||||
for j = 0; j < num_histograms; j++ {
|
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < int(num_histograms); i++ {
|
||||
cost[i] = 0
|
||||
}
|
||||
for i := 0; i < int(length*bitmaplen); i++ {
|
||||
switch_signal[i] = 0
|
||||
}
|
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference
|
||||
between the minimum cost of arriving at the current byte position using
|
||||
entropy code k, and the minimum cost of arriving at the current byte
|
||||
position. This difference is capped at the block switch cost, and if it
|
||||
reaches block switch cost, it means that when we trace back from the last
|
||||
position, we need to switch here. */
|
||||
for i = 0; i < length; i++ {
|
||||
var byte_ix uint = i
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
|
||||
var min_cost float64 = 1e99
|
||||
var block_switch_cost float64 = block_switch_bitcost
|
||||
var k uint
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */
|
||||
cost[k] += insert_cost[insert_cost_ix+k]
|
||||
|
||||
if cost[k] < min_cost {
|
||||
min_cost = cost[k]
|
||||
block_id[byte_ix] = byte(k)
|
||||
}
|
||||
}
|
||||
|
||||
/* More blocks for the beginning. */
|
||||
if byte_ix < 2000 {
|
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
|
||||
}
|
||||
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
cost[k] -= min_cost
|
||||
if cost[k] >= block_switch_cost {
|
||||
var mask byte = byte(1 << (k & 7))
|
||||
cost[k] = block_switch_cost
|
||||
assert(k>>3 < bitmaplen)
|
||||
switch_signal[ix+(k>>3)] |= mask
|
||||
/* Trace back from the last position and switch at the marked places. */
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
var byte_ix uint = length - 1
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var cur_id byte = block_id[byte_ix]
|
||||
for byte_ix > 0 {
|
||||
var mask byte = byte(1 << (cur_id & 7))
|
||||
assert(uint(cur_id)>>3 < bitmaplen)
|
||||
byte_ix--
|
||||
ix -= bitmaplen
|
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
|
||||
if cur_id != block_id[byte_ix] {
|
||||
cur_id = block_id[byte_ix]
|
||||
num_blocks++
|
||||
}
|
||||
}
|
||||
|
||||
block_id[byte_ix] = cur_id
|
||||
}
|
||||
}
|
||||
|
||||
return num_blocks
|
||||
}
|
||||
|
||||
var remapBlockIdsCommand_kInvalidId uint16 = 256
|
||||
|
||||
func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
|
||||
var next_id uint16 = 0
|
||||
var i uint
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
new_id[i] = remapBlockIdsCommand_kInvalidId
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId {
|
||||
new_id[block_ids[i]] = next_id
|
||||
next_id++
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
block_ids[i] = byte(new_id[block_ids[i]])
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
}
|
||||
|
||||
assert(uint(next_id) <= num_histograms)
|
||||
return uint(next_id)
|
||||
}
|
||||
|
||||
func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
|
||||
var i uint
|
||||
clearHistogramsCommand(histograms, num_histograms)
|
||||
for i = 0; i < length; i++ {
|
||||
histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
|
||||
}
|
||||
}
|
||||
|
||||
var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
|
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks)
|
||||
var block_lengths []uint32 = make([]uint32, num_blocks)
|
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
|
||||
var all_histograms_size uint = 0
|
||||
var all_histograms_capacity uint = expected_num_clusters
|
||||
var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
|
||||
var cluster_size_size uint = 0
|
||||
var cluster_size_capacity uint = expected_num_clusters
|
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
|
||||
var num_clusters uint = 0
|
||||
var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
|
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
|
||||
var pairs_capacity uint = max_num_pairs + 1
|
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
|
||||
var pos uint = 0
|
||||
var clusters []uint32
|
||||
var num_final_clusters uint
|
||||
var new_index []uint32
|
||||
var i uint
|
||||
var sizes = [histogramsPerBatch]uint32{0}
|
||||
var new_clusters = [histogramsPerBatch]uint32{0}
|
||||
var symbols = [histogramsPerBatch]uint32{0}
|
||||
var remap = [histogramsPerBatch]uint32{0}
|
||||
|
||||
for i := 0; i < int(num_blocks); i++ {
|
||||
block_lengths[i] = 0
|
||||
}
|
||||
{
|
||||
var block_idx uint = 0
|
||||
for i = 0; i < length; i++ {
|
||||
assert(block_idx < num_blocks)
|
||||
block_lengths[block_idx]++
|
||||
if i+1 == length || block_ids[i] != block_ids[i+1] {
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
assert(block_idx == num_blocks)
|
||||
}
|
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch {
|
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
var k uint
|
||||
histogramClearCommand(&histograms[j])
|
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
|
||||
histogramAddCommand(&histograms[j], uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
histograms[j].bit_cost_ = populationCostCommand(&histograms[j])
|
||||
new_clusters[j] = uint32(j)
|
||||
symbols[j] = uint32(j)
|
||||
sizes[j] = 1
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
|
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
|
||||
var _new_size uint
|
||||
if all_histograms_capacity == 0 {
|
||||
_new_size = all_histograms_size + num_new_clusters
|
||||
} else {
|
||||
_new_size = all_histograms_capacity
|
||||
}
|
||||
var new_array []histogramCommand
|
||||
for _new_size < (all_histograms_size + num_new_clusters) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramCommand, _new_size)
|
||||
if all_histograms_capacity != 0 {
|
||||
copy(new_array, all_histograms[:all_histograms_capacity])
|
||||
}
|
||||
|
||||
all_histograms = new_array
|
||||
all_histograms_capacity = _new_size
|
||||
}
|
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
|
||||
for j = 0; j < num_new_clusters; j++ {
|
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
|
||||
all_histograms_size++
|
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
|
||||
cluster_size_size++
|
||||
remap[new_clusters[j]] = uint32(j)
|
||||
}
|
||||
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
|
||||
}
|
||||
|
||||
num_clusters += num_new_clusters
|
||||
assert(num_clusters == cluster_size_size)
|
||||
assert(num_clusters == all_histograms_size)
|
||||
}
|
||||
|
||||
histograms = nil
|
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < max_num_pairs+1 {
|
||||
pairs = nil
|
||||
pairs = make([]histogramPair, (max_num_pairs + 1))
|
||||
}
|
||||
|
||||
clusters = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
clusters[i] = uint32(i)
|
||||
}
|
||||
|
||||
num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
new_index = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
new_index[i] = clusterBlocksCommand_kInvalidIndex
|
||||
}
|
||||
pos = 0
|
||||
{
|
||||
var next_index uint32 = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
var histo histogramCommand
|
||||
var j uint
|
||||
var best_out uint32
|
||||
var best_bits float64
|
||||
histogramClearCommand(&histo)
|
||||
for j = 0; uint32(j) < block_lengths[i]; j++ {
|
||||
histogramAddCommand(&histo, uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
best_out = histogram_symbols[0]
|
||||
} else {
|
||||
best_out = histogram_symbols[i-1]
|
||||
}
|
||||
best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out])
|
||||
for j = 0; j < num_final_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
histogram_symbols[i] = best_out
|
||||
if new_index[best_out] == clusterBlocksCommand_kInvalidIndex {
|
||||
new_index[best_out] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusters = nil
|
||||
all_histograms = nil
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
|
||||
{
|
||||
var cur_length uint32 = 0
|
||||
var block_idx uint = 0
|
||||
var max_type byte = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
cur_length += block_lengths[i]
|
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
|
||||
var id byte = byte(new_index[histogram_symbols[i]])
|
||||
split.types[block_idx] = id
|
||||
split.lengths[block_idx] = cur_length
|
||||
max_type = brotli_max_uint8_t(max_type, id)
|
||||
cur_length = 0
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
split.num_blocks = block_idx
|
||||
split.num_types = uint(max_type) + 1
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
block_lengths = nil
|
||||
histogram_symbols = nil
|
||||
}
|
||||
|
||||
func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
|
||||
length := uint(len(data))
|
||||
var data_size uint = histogramDataSizeCommand()
|
||||
var num_histograms uint = length/literals_per_histogram + 1
|
||||
var histograms []histogramCommand
|
||||
if num_histograms > max_histograms {
|
||||
num_histograms = max_histograms
|
||||
}
|
||||
|
||||
if length == 0 {
|
||||
split.num_types = 1
|
||||
return
|
||||
} else if length < kMinLengthForBlockSplitting {
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
|
||||
split.num_types = 1
|
||||
split.types[split.num_blocks] = 0
|
||||
split.lengths[split.num_blocks] = uint32(length)
|
||||
split.num_blocks++
|
||||
return
|
||||
}
|
||||
|
||||
histograms = make([]histogramCommand, num_histograms)
|
||||
|
||||
/* Find good entropy codes. */
|
||||
initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
|
||||
refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
{
|
||||
var block_ids []byte = make([]byte, length)
|
||||
var num_blocks uint = 0
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
|
||||
var cost []float64 = make([]float64, num_histograms)
|
||||
var switch_signal []byte = make([]byte, (length * bitmaplen))
|
||||
var new_id []uint16 = make([]uint16, num_histograms)
|
||||
var iters uint
|
||||
if params.quality < hqZopflificationQuality {
|
||||
iters = 3
|
||||
} else {
|
||||
iters = 10
|
||||
}
|
||||
/* Find a good path through literals with the good entropy codes. */
|
||||
|
||||
var i uint
|
||||
for i = 0; i < iters; i++ {
|
||||
num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
|
||||
num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms)
|
||||
buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms)
|
||||
}
|
||||
|
||||
insert_cost = nil
|
||||
cost = nil
|
||||
switch_signal = nil
|
||||
new_id = nil
|
||||
histograms = nil
|
||||
clusterBlocksCommand(data, length, num_blocks, block_ids, split)
|
||||
block_ids = nil
|
||||
}
|
||||
}
|
||||
433
vendor/github.com/andybalholm/brotli/block_splitter_distance.go
generated
vendored
Normal file
433
vendor/github.com/andybalholm/brotli/block_splitter_distance.go
generated
vendored
Normal file
@ -0,0 +1,433 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
|
||||
var seed uint32 = 7
|
||||
var block_length uint = length / num_histograms
|
||||
var i uint
|
||||
clearHistogramsDistance(histograms, num_histograms)
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
var pos uint = length * i / num_histograms
|
||||
if i != 0 {
|
||||
pos += uint(myRand(&seed) % uint32(block_length))
|
||||
}
|
||||
|
||||
if pos+stride >= length {
|
||||
pos = length - stride - 1
|
||||
}
|
||||
|
||||
histogramAddVectorDistance(&histograms[i], data[pos:], stride)
|
||||
}
|
||||
}
|
||||
|
||||
func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
|
||||
var pos uint = 0
|
||||
if stride >= length {
|
||||
stride = length
|
||||
} else {
|
||||
pos = uint(myRand(seed) % uint32(length-stride+1))
|
||||
}
|
||||
|
||||
histogramAddVectorDistance(sample, data[pos:], stride)
|
||||
}
|
||||
|
||||
func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
|
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
|
||||
var seed uint32 = 7
|
||||
var iter uint
|
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
|
||||
for iter = 0; iter < iters; iter++ {
|
||||
var sample histogramDistance
|
||||
histogramClearDistance(&sample)
|
||||
randomSampleDistance(&seed, data, length, stride, &sample)
|
||||
histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
|
||||
}
|
||||
}
|
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element
|
||||
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
||||
Returns the number of blocks, i.e. one plus the number of block switches. */
|
||||
func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
|
||||
var data_size uint = histogramDataSizeDistance()
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var num_blocks uint = 1
|
||||
var i uint
|
||||
var j uint
|
||||
assert(num_histograms <= 256)
|
||||
if num_histograms <= 1 {
|
||||
for i = 0; i < length; i++ {
|
||||
block_id[i] = 0
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ {
|
||||
insert_cost[i] = 0
|
||||
}
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
|
||||
}
|
||||
|
||||
for i = data_size; i != 0; {
|
||||
i--
|
||||
for j = 0; j < num_histograms; j++ {
|
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < int(num_histograms); i++ {
|
||||
cost[i] = 0
|
||||
}
|
||||
for i := 0; i < int(length*bitmaplen); i++ {
|
||||
switch_signal[i] = 0
|
||||
}
|
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference
|
||||
between the minimum cost of arriving at the current byte position using
|
||||
entropy code k, and the minimum cost of arriving at the current byte
|
||||
position. This difference is capped at the block switch cost, and if it
|
||||
reaches block switch cost, it means that when we trace back from the last
|
||||
position, we need to switch here. */
|
||||
for i = 0; i < length; i++ {
|
||||
var byte_ix uint = i
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
|
||||
var min_cost float64 = 1e99
|
||||
var block_switch_cost float64 = block_switch_bitcost
|
||||
var k uint
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */
|
||||
cost[k] += insert_cost[insert_cost_ix+k]
|
||||
|
||||
if cost[k] < min_cost {
|
||||
min_cost = cost[k]
|
||||
block_id[byte_ix] = byte(k)
|
||||
}
|
||||
}
|
||||
|
||||
/* More blocks for the beginning. */
|
||||
if byte_ix < 2000 {
|
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
|
||||
}
|
||||
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
cost[k] -= min_cost
|
||||
if cost[k] >= block_switch_cost {
|
||||
var mask byte = byte(1 << (k & 7))
|
||||
cost[k] = block_switch_cost
|
||||
assert(k>>3 < bitmaplen)
|
||||
switch_signal[ix+(k>>3)] |= mask
|
||||
/* Trace back from the last position and switch at the marked places. */
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
var byte_ix uint = length - 1
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var cur_id byte = block_id[byte_ix]
|
||||
for byte_ix > 0 {
|
||||
var mask byte = byte(1 << (cur_id & 7))
|
||||
assert(uint(cur_id)>>3 < bitmaplen)
|
||||
byte_ix--
|
||||
ix -= bitmaplen
|
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
|
||||
if cur_id != block_id[byte_ix] {
|
||||
cur_id = block_id[byte_ix]
|
||||
num_blocks++
|
||||
}
|
||||
}
|
||||
|
||||
block_id[byte_ix] = cur_id
|
||||
}
|
||||
}
|
||||
|
||||
return num_blocks
|
||||
}
|
||||
|
||||
var remapBlockIdsDistance_kInvalidId uint16 = 256
|
||||
|
||||
func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
|
||||
var next_id uint16 = 0
|
||||
var i uint
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
new_id[i] = remapBlockIdsDistance_kInvalidId
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId {
|
||||
new_id[block_ids[i]] = next_id
|
||||
next_id++
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
block_ids[i] = byte(new_id[block_ids[i]])
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
}
|
||||
|
||||
assert(uint(next_id) <= num_histograms)
|
||||
return uint(next_id)
|
||||
}
|
||||
|
||||
func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
|
||||
var i uint
|
||||
clearHistogramsDistance(histograms, num_histograms)
|
||||
for i = 0; i < length; i++ {
|
||||
histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
|
||||
}
|
||||
}
|
||||
|
||||
var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
|
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks)
|
||||
var block_lengths []uint32 = make([]uint32, num_blocks)
|
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
|
||||
var all_histograms_size uint = 0
|
||||
var all_histograms_capacity uint = expected_num_clusters
|
||||
var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
|
||||
var cluster_size_size uint = 0
|
||||
var cluster_size_capacity uint = expected_num_clusters
|
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
|
||||
var num_clusters uint = 0
|
||||
var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
|
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
|
||||
var pairs_capacity uint = max_num_pairs + 1
|
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
|
||||
var pos uint = 0
|
||||
var clusters []uint32
|
||||
var num_final_clusters uint
|
||||
var new_index []uint32
|
||||
var i uint
|
||||
var sizes = [histogramsPerBatch]uint32{0}
|
||||
var new_clusters = [histogramsPerBatch]uint32{0}
|
||||
var symbols = [histogramsPerBatch]uint32{0}
|
||||
var remap = [histogramsPerBatch]uint32{0}
|
||||
|
||||
for i := 0; i < int(num_blocks); i++ {
|
||||
block_lengths[i] = 0
|
||||
}
|
||||
{
|
||||
var block_idx uint = 0
|
||||
for i = 0; i < length; i++ {
|
||||
assert(block_idx < num_blocks)
|
||||
block_lengths[block_idx]++
|
||||
if i+1 == length || block_ids[i] != block_ids[i+1] {
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
assert(block_idx == num_blocks)
|
||||
}
|
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch {
|
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
var k uint
|
||||
histogramClearDistance(&histograms[j])
|
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
|
||||
histogramAddDistance(&histograms[j], uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
histograms[j].bit_cost_ = populationCostDistance(&histograms[j])
|
||||
new_clusters[j] = uint32(j)
|
||||
symbols[j] = uint32(j)
|
||||
sizes[j] = 1
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
|
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
|
||||
var _new_size uint
|
||||
if all_histograms_capacity == 0 {
|
||||
_new_size = all_histograms_size + num_new_clusters
|
||||
} else {
|
||||
_new_size = all_histograms_capacity
|
||||
}
|
||||
var new_array []histogramDistance
|
||||
for _new_size < (all_histograms_size + num_new_clusters) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramDistance, _new_size)
|
||||
if all_histograms_capacity != 0 {
|
||||
copy(new_array, all_histograms[:all_histograms_capacity])
|
||||
}
|
||||
|
||||
all_histograms = new_array
|
||||
all_histograms_capacity = _new_size
|
||||
}
|
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
|
||||
for j = 0; j < num_new_clusters; j++ {
|
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
|
||||
all_histograms_size++
|
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
|
||||
cluster_size_size++
|
||||
remap[new_clusters[j]] = uint32(j)
|
||||
}
|
||||
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
|
||||
}
|
||||
|
||||
num_clusters += num_new_clusters
|
||||
assert(num_clusters == cluster_size_size)
|
||||
assert(num_clusters == all_histograms_size)
|
||||
}
|
||||
|
||||
histograms = nil
|
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < max_num_pairs+1 {
|
||||
pairs = nil
|
||||
pairs = make([]histogramPair, (max_num_pairs + 1))
|
||||
}
|
||||
|
||||
clusters = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
clusters[i] = uint32(i)
|
||||
}
|
||||
|
||||
num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
new_index = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
new_index[i] = clusterBlocksDistance_kInvalidIndex
|
||||
}
|
||||
pos = 0
|
||||
{
|
||||
var next_index uint32 = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
var histo histogramDistance
|
||||
var j uint
|
||||
var best_out uint32
|
||||
var best_bits float64
|
||||
histogramClearDistance(&histo)
|
||||
for j = 0; uint32(j) < block_lengths[i]; j++ {
|
||||
histogramAddDistance(&histo, uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
best_out = histogram_symbols[0]
|
||||
} else {
|
||||
best_out = histogram_symbols[i-1]
|
||||
}
|
||||
best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out])
|
||||
for j = 0; j < num_final_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
histogram_symbols[i] = best_out
|
||||
if new_index[best_out] == clusterBlocksDistance_kInvalidIndex {
|
||||
new_index[best_out] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusters = nil
|
||||
all_histograms = nil
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
|
||||
{
|
||||
var cur_length uint32 = 0
|
||||
var block_idx uint = 0
|
||||
var max_type byte = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
cur_length += block_lengths[i]
|
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
|
||||
var id byte = byte(new_index[histogram_symbols[i]])
|
||||
split.types[block_idx] = id
|
||||
split.lengths[block_idx] = cur_length
|
||||
max_type = brotli_max_uint8_t(max_type, id)
|
||||
cur_length = 0
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
split.num_blocks = block_idx
|
||||
split.num_types = uint(max_type) + 1
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
block_lengths = nil
|
||||
histogram_symbols = nil
|
||||
}
|
||||
|
||||
func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
|
||||
var data_size uint = histogramDataSizeDistance()
|
||||
var num_histograms uint = length/literals_per_histogram + 1
|
||||
var histograms []histogramDistance
|
||||
if num_histograms > max_histograms {
|
||||
num_histograms = max_histograms
|
||||
}
|
||||
|
||||
if length == 0 {
|
||||
split.num_types = 1
|
||||
return
|
||||
} else if length < kMinLengthForBlockSplitting {
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
|
||||
split.num_types = 1
|
||||
split.types[split.num_blocks] = 0
|
||||
split.lengths[split.num_blocks] = uint32(length)
|
||||
split.num_blocks++
|
||||
return
|
||||
}
|
||||
|
||||
histograms = make([]histogramDistance, num_histograms)
|
||||
|
||||
/* Find good entropy codes. */
|
||||
initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
|
||||
refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
{
|
||||
var block_ids []byte = make([]byte, length)
|
||||
var num_blocks uint = 0
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
|
||||
var cost []float64 = make([]float64, num_histograms)
|
||||
var switch_signal []byte = make([]byte, (length * bitmaplen))
|
||||
var new_id []uint16 = make([]uint16, num_histograms)
|
||||
var iters uint
|
||||
if params.quality < hqZopflificationQuality {
|
||||
iters = 3
|
||||
} else {
|
||||
iters = 10
|
||||
}
|
||||
/* Find a good path through literals with the good entropy codes. */
|
||||
|
||||
var i uint
|
||||
for i = 0; i < iters; i++ {
|
||||
num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
|
||||
num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms)
|
||||
buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms)
|
||||
}
|
||||
|
||||
insert_cost = nil
|
||||
cost = nil
|
||||
switch_signal = nil
|
||||
new_id = nil
|
||||
histograms = nil
|
||||
clusterBlocksDistance(data, length, num_blocks, block_ids, split)
|
||||
block_ids = nil
|
||||
}
|
||||
}
|
||||
433
vendor/github.com/andybalholm/brotli/block_splitter_literal.go
generated
vendored
Normal file
433
vendor/github.com/andybalholm/brotli/block_splitter_literal.go
generated
vendored
Normal file
@ -0,0 +1,433 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
|
||||
var seed uint32 = 7
|
||||
var block_length uint = length / num_histograms
|
||||
var i uint
|
||||
clearHistogramsLiteral(histograms, num_histograms)
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
var pos uint = length * i / num_histograms
|
||||
if i != 0 {
|
||||
pos += uint(myRand(&seed) % uint32(block_length))
|
||||
}
|
||||
|
||||
if pos+stride >= length {
|
||||
pos = length - stride - 1
|
||||
}
|
||||
|
||||
histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
|
||||
}
|
||||
}
|
||||
|
||||
func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
|
||||
var pos uint = 0
|
||||
if stride >= length {
|
||||
stride = length
|
||||
} else {
|
||||
pos = uint(myRand(seed) % uint32(length-stride+1))
|
||||
}
|
||||
|
||||
histogramAddVectorLiteral(sample, data[pos:], stride)
|
||||
}
|
||||
|
||||
func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
|
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
|
||||
var seed uint32 = 7
|
||||
var iter uint
|
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
|
||||
for iter = 0; iter < iters; iter++ {
|
||||
var sample histogramLiteral
|
||||
histogramClearLiteral(&sample)
|
||||
randomSampleLiteral(&seed, data, length, stride, &sample)
|
||||
histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
|
||||
}
|
||||
}
|
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element
|
||||
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
||||
Returns the number of blocks, i.e. one plus the number of block switches. */
|
||||
func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
|
||||
var data_size uint = histogramDataSizeLiteral()
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var num_blocks uint = 1
|
||||
var i uint
|
||||
var j uint
|
||||
assert(num_histograms <= 256)
|
||||
if num_histograms <= 1 {
|
||||
for i = 0; i < length; i++ {
|
||||
block_id[i] = 0
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ {
|
||||
insert_cost[i] = 0
|
||||
}
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
|
||||
}
|
||||
|
||||
for i = data_size; i != 0; {
|
||||
i--
|
||||
for j = 0; j < num_histograms; j++ {
|
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < int(num_histograms); i++ {
|
||||
cost[i] = 0
|
||||
}
|
||||
for i := 0; i < int(length*bitmaplen); i++ {
|
||||
switch_signal[i] = 0
|
||||
}
|
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference
|
||||
between the minimum cost of arriving at the current byte position using
|
||||
entropy code k, and the minimum cost of arriving at the current byte
|
||||
position. This difference is capped at the block switch cost, and if it
|
||||
reaches block switch cost, it means that when we trace back from the last
|
||||
position, we need to switch here. */
|
||||
for i = 0; i < length; i++ {
|
||||
var byte_ix uint = i
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
|
||||
var min_cost float64 = 1e99
|
||||
var block_switch_cost float64 = block_switch_bitcost
|
||||
var k uint
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */
|
||||
cost[k] += insert_cost[insert_cost_ix+k]
|
||||
|
||||
if cost[k] < min_cost {
|
||||
min_cost = cost[k]
|
||||
block_id[byte_ix] = byte(k)
|
||||
}
|
||||
}
|
||||
|
||||
/* More blocks for the beginning. */
|
||||
if byte_ix < 2000 {
|
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
|
||||
}
|
||||
|
||||
for k = 0; k < num_histograms; k++ {
|
||||
cost[k] -= min_cost
|
||||
if cost[k] >= block_switch_cost {
|
||||
var mask byte = byte(1 << (k & 7))
|
||||
cost[k] = block_switch_cost
|
||||
assert(k>>3 < bitmaplen)
|
||||
switch_signal[ix+(k>>3)] |= mask
|
||||
/* Trace back from the last position and switch at the marked places. */
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
var byte_ix uint = length - 1
|
||||
var ix uint = byte_ix * bitmaplen
|
||||
var cur_id byte = block_id[byte_ix]
|
||||
for byte_ix > 0 {
|
||||
var mask byte = byte(1 << (cur_id & 7))
|
||||
assert(uint(cur_id)>>3 < bitmaplen)
|
||||
byte_ix--
|
||||
ix -= bitmaplen
|
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
|
||||
if cur_id != block_id[byte_ix] {
|
||||
cur_id = block_id[byte_ix]
|
||||
num_blocks++
|
||||
}
|
||||
}
|
||||
|
||||
block_id[byte_ix] = cur_id
|
||||
}
|
||||
}
|
||||
|
||||
return num_blocks
|
||||
}
|
||||
|
||||
var remapBlockIdsLiteral_kInvalidId uint16 = 256
|
||||
|
||||
func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
|
||||
var next_id uint16 = 0
|
||||
var i uint
|
||||
for i = 0; i < num_histograms; i++ {
|
||||
new_id[i] = remapBlockIdsLiteral_kInvalidId
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId {
|
||||
new_id[block_ids[i]] = next_id
|
||||
next_id++
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < length; i++ {
|
||||
block_ids[i] = byte(new_id[block_ids[i]])
|
||||
assert(uint(block_ids[i]) < num_histograms)
|
||||
}
|
||||
|
||||
assert(uint(next_id) <= num_histograms)
|
||||
return uint(next_id)
|
||||
}
|
||||
|
||||
func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
|
||||
var i uint
|
||||
clearHistogramsLiteral(histograms, num_histograms)
|
||||
for i = 0; i < length; i++ {
|
||||
histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
|
||||
}
|
||||
}
|
||||
|
||||
var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
|
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks)
|
||||
var block_lengths []uint32 = make([]uint32, num_blocks)
|
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
|
||||
var all_histograms_size uint = 0
|
||||
var all_histograms_capacity uint = expected_num_clusters
|
||||
var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
|
||||
var cluster_size_size uint = 0
|
||||
var cluster_size_capacity uint = expected_num_clusters
|
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
|
||||
var num_clusters uint = 0
|
||||
var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
|
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
|
||||
var pairs_capacity uint = max_num_pairs + 1
|
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
|
||||
var pos uint = 0
|
||||
var clusters []uint32
|
||||
var num_final_clusters uint
|
||||
var new_index []uint32
|
||||
var i uint
|
||||
var sizes = [histogramsPerBatch]uint32{0}
|
||||
var new_clusters = [histogramsPerBatch]uint32{0}
|
||||
var symbols = [histogramsPerBatch]uint32{0}
|
||||
var remap = [histogramsPerBatch]uint32{0}
|
||||
|
||||
for i := 0; i < int(num_blocks); i++ {
|
||||
block_lengths[i] = 0
|
||||
}
|
||||
{
|
||||
var block_idx uint = 0
|
||||
for i = 0; i < length; i++ {
|
||||
assert(block_idx < num_blocks)
|
||||
block_lengths[block_idx]++
|
||||
if i+1 == length || block_ids[i] != block_ids[i+1] {
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
assert(block_idx == num_blocks)
|
||||
}
|
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch {
|
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
var k uint
|
||||
histogramClearLiteral(&histograms[j])
|
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
|
||||
histogramAddLiteral(&histograms[j], uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
histograms[j].bit_cost_ = populationCostLiteral(&histograms[j])
|
||||
new_clusters[j] = uint32(j)
|
||||
symbols[j] = uint32(j)
|
||||
sizes[j] = 1
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
|
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
|
||||
var _new_size uint
|
||||
if all_histograms_capacity == 0 {
|
||||
_new_size = all_histograms_size + num_new_clusters
|
||||
} else {
|
||||
_new_size = all_histograms_capacity
|
||||
}
|
||||
var new_array []histogramLiteral
|
||||
for _new_size < (all_histograms_size + num_new_clusters) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramLiteral, _new_size)
|
||||
if all_histograms_capacity != 0 {
|
||||
copy(new_array, all_histograms[:all_histograms_capacity])
|
||||
}
|
||||
|
||||
all_histograms = new_array
|
||||
all_histograms_capacity = _new_size
|
||||
}
|
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
|
||||
for j = 0; j < num_new_clusters; j++ {
|
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
|
||||
all_histograms_size++
|
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
|
||||
cluster_size_size++
|
||||
remap[new_clusters[j]] = uint32(j)
|
||||
}
|
||||
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
|
||||
}
|
||||
|
||||
num_clusters += num_new_clusters
|
||||
assert(num_clusters == cluster_size_size)
|
||||
assert(num_clusters == all_histograms_size)
|
||||
}
|
||||
|
||||
histograms = nil
|
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < max_num_pairs+1 {
|
||||
pairs = nil
|
||||
pairs = make([]histogramPair, (max_num_pairs + 1))
|
||||
}
|
||||
|
||||
clusters = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
clusters[i] = uint32(i)
|
||||
}
|
||||
|
||||
num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
new_index = make([]uint32, num_clusters)
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
new_index[i] = clusterBlocksLiteral_kInvalidIndex
|
||||
}
|
||||
pos = 0
|
||||
{
|
||||
var next_index uint32 = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
var histo histogramLiteral
|
||||
var j uint
|
||||
var best_out uint32
|
||||
var best_bits float64
|
||||
histogramClearLiteral(&histo)
|
||||
for j = 0; uint32(j) < block_lengths[i]; j++ {
|
||||
histogramAddLiteral(&histo, uint(data[pos]))
|
||||
pos++
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
best_out = histogram_symbols[0]
|
||||
} else {
|
||||
best_out = histogram_symbols[i-1]
|
||||
}
|
||||
best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out])
|
||||
for j = 0; j < num_final_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
histogram_symbols[i] = best_out
|
||||
if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex {
|
||||
new_index[best_out] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusters = nil
|
||||
all_histograms = nil
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
|
||||
{
|
||||
var cur_length uint32 = 0
|
||||
var block_idx uint = 0
|
||||
var max_type byte = 0
|
||||
for i = 0; i < num_blocks; i++ {
|
||||
cur_length += block_lengths[i]
|
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
|
||||
var id byte = byte(new_index[histogram_symbols[i]])
|
||||
split.types[block_idx] = id
|
||||
split.lengths[block_idx] = cur_length
|
||||
max_type = brotli_max_uint8_t(max_type, id)
|
||||
cur_length = 0
|
||||
block_idx++
|
||||
}
|
||||
}
|
||||
|
||||
split.num_blocks = block_idx
|
||||
split.num_types = uint(max_type) + 1
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
block_lengths = nil
|
||||
histogram_symbols = nil
|
||||
}
|
||||
|
||||
func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
|
||||
var data_size uint = histogramDataSizeLiteral()
|
||||
var num_histograms uint = length/literals_per_histogram + 1
|
||||
var histograms []histogramLiteral
|
||||
if num_histograms > max_histograms {
|
||||
num_histograms = max_histograms
|
||||
}
|
||||
|
||||
if length == 0 {
|
||||
split.num_types = 1
|
||||
return
|
||||
} else if length < kMinLengthForBlockSplitting {
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
|
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
|
||||
split.num_types = 1
|
||||
split.types[split.num_blocks] = 0
|
||||
split.lengths[split.num_blocks] = uint32(length)
|
||||
split.num_blocks++
|
||||
return
|
||||
}
|
||||
|
||||
histograms = make([]histogramLiteral, num_histograms)
|
||||
|
||||
/* Find good entropy codes. */
|
||||
initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
|
||||
refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
|
||||
{
|
||||
var block_ids []byte = make([]byte, length)
|
||||
var num_blocks uint = 0
|
||||
var bitmaplen uint = (num_histograms + 7) >> 3
|
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
|
||||
var cost []float64 = make([]float64, num_histograms)
|
||||
var switch_signal []byte = make([]byte, (length * bitmaplen))
|
||||
var new_id []uint16 = make([]uint16, num_histograms)
|
||||
var iters uint
|
||||
if params.quality < hqZopflificationQuality {
|
||||
iters = 3
|
||||
} else {
|
||||
iters = 10
|
||||
}
|
||||
/* Find a good path through literals with the good entropy codes. */
|
||||
|
||||
var i uint
|
||||
for i = 0; i < iters; i++ {
|
||||
num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
|
||||
num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms)
|
||||
buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms)
|
||||
}
|
||||
|
||||
insert_cost = nil
|
||||
cost = nil
|
||||
switch_signal = nil
|
||||
new_id = nil
|
||||
histograms = nil
|
||||
clusterBlocksLiteral(data, length, num_blocks, block_ids, split)
|
||||
block_ids = nil
|
||||
}
|
||||
}
|
||||
1300
vendor/github.com/andybalholm/brotli/brotli_bit_stream.go
generated
vendored
Normal file
1300
vendor/github.com/andybalholm/brotli/brotli_bit_stream.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
30
vendor/github.com/andybalholm/brotli/cluster.go
generated
vendored
Normal file
30
vendor/github.com/andybalholm/brotli/cluster.go
generated
vendored
Normal file
@ -0,0 +1,30 @@
|
||||
package brotli
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Functions for clustering similar histograms together. */
|
||||
|
||||
type histogramPair struct {
|
||||
idx1 uint32
|
||||
idx2 uint32
|
||||
cost_combo float64
|
||||
cost_diff float64
|
||||
}
|
||||
|
||||
func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
|
||||
if p1.cost_diff != p2.cost_diff {
|
||||
return p1.cost_diff > p2.cost_diff
|
||||
}
|
||||
|
||||
return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1)
|
||||
}
|
||||
|
||||
/* Returns entropy reduction of the context map when we combine two clusters. */
|
||||
func clusterCostDiff(size_a uint, size_b uint) float64 {
|
||||
var size_c uint = size_a + size_b
|
||||
return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c)
|
||||
}
|
||||
164
vendor/github.com/andybalholm/brotli/cluster_command.go
generated
vendored
Normal file
164
vendor/github.com/andybalholm/brotli/cluster_command.go
generated
vendored
Normal file
@ -0,0 +1,164 @@
|
||||
package brotli
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
|
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
|
||||
func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
|
||||
var is_good_pair bool = false
|
||||
var p histogramPair
|
||||
p.idx2 = 0
|
||||
p.idx1 = p.idx2
|
||||
p.cost_combo = 0
|
||||
p.cost_diff = p.cost_combo
|
||||
if idx1 == idx2 {
|
||||
return
|
||||
}
|
||||
|
||||
if idx2 < idx1 {
|
||||
var t uint32 = idx2
|
||||
idx2 = idx1
|
||||
idx1 = t
|
||||
}
|
||||
|
||||
p.idx1 = idx1
|
||||
p.idx2 = idx2
|
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
|
||||
p.cost_diff -= out[idx1].bit_cost_
|
||||
p.cost_diff -= out[idx2].bit_cost_
|
||||
|
||||
if out[idx1].total_count_ == 0 {
|
||||
p.cost_combo = out[idx2].bit_cost_
|
||||
is_good_pair = true
|
||||
} else if out[idx2].total_count_ == 0 {
|
||||
p.cost_combo = out[idx1].bit_cost_
|
||||
is_good_pair = true
|
||||
} else {
|
||||
var threshold float64
|
||||
if *num_pairs == 0 {
|
||||
threshold = 1e99
|
||||
} else {
|
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
|
||||
}
|
||||
var combo histogramCommand = out[idx1]
|
||||
var cost_combo float64
|
||||
histogramAddHistogramCommand(&combo, &out[idx2])
|
||||
cost_combo = populationCostCommand(&combo)
|
||||
if cost_combo < threshold-p.cost_diff {
|
||||
p.cost_combo = cost_combo
|
||||
is_good_pair = true
|
||||
}
|
||||
}
|
||||
|
||||
if is_good_pair {
|
||||
p.cost_diff += p.cost_combo
|
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = pairs[0]
|
||||
(*num_pairs)++
|
||||
}
|
||||
|
||||
pairs[0] = p
|
||||
} else if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = p
|
||||
(*num_pairs)++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
|
||||
var cost_diff_threshold float64 = 0.0
|
||||
var min_cluster_size uint = 1
|
||||
var num_pairs uint = 0
|
||||
{
|
||||
/* We maintain a vector of histogram pairs, with the property that the pair
|
||||
with the maximum bit cost reduction is the first. */
|
||||
var idx1 uint
|
||||
for idx1 = 0; idx1 < num_clusters; idx1++ {
|
||||
var idx2 uint
|
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
|
||||
compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for num_clusters > min_cluster_size {
|
||||
var best_idx1 uint32
|
||||
var best_idx2 uint32
|
||||
var i uint
|
||||
if pairs[0].cost_diff >= cost_diff_threshold {
|
||||
cost_diff_threshold = 1e99
|
||||
min_cluster_size = max_clusters
|
||||
continue
|
||||
}
|
||||
|
||||
/* Take the best pair from the top of heap. */
|
||||
best_idx1 = pairs[0].idx1
|
||||
|
||||
best_idx2 = pairs[0].idx2
|
||||
histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
|
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo
|
||||
cluster_size[best_idx1] += cluster_size[best_idx2]
|
||||
for i = 0; i < symbols_size; i++ {
|
||||
if symbols[i] == best_idx2 {
|
||||
symbols[i] = best_idx1
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
if clusters[i] == best_idx2 {
|
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
num_clusters--
|
||||
{
|
||||
/* Remove pairs intersecting the just combined best pair. */
|
||||
var copy_to_idx uint = 0
|
||||
for i = 0; i < num_pairs; i++ {
|
||||
var p *histogramPair = &pairs[i]
|
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
|
||||
/* Remove invalid pair from the queue. */
|
||||
continue
|
||||
}
|
||||
|
||||
if histogramPairIsLess(&pairs[0], p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
var front histogramPair = pairs[0]
|
||||
pairs[0] = *p
|
||||
pairs[copy_to_idx] = front
|
||||
} else {
|
||||
pairs[copy_to_idx] = *p
|
||||
}
|
||||
|
||||
copy_to_idx++
|
||||
}
|
||||
|
||||
num_pairs = copy_to_idx
|
||||
}
|
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
|
||||
return num_clusters
|
||||
}
|
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
|
||||
func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 {
|
||||
if histogram.total_count_ == 0 {
|
||||
return 0.0
|
||||
} else {
|
||||
var tmp histogramCommand = *histogram
|
||||
histogramAddHistogramCommand(&tmp, candidate)
|
||||
return populationCostCommand(&tmp) - candidate.bit_cost_
|
||||
}
|
||||
}
|
||||
326
vendor/github.com/andybalholm/brotli/cluster_distance.go
generated
vendored
Normal file
326
vendor/github.com/andybalholm/brotli/cluster_distance.go
generated
vendored
Normal file
@ -0,0 +1,326 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
|
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
|
||||
func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
|
||||
var is_good_pair bool = false
|
||||
var p histogramPair
|
||||
p.idx2 = 0
|
||||
p.idx1 = p.idx2
|
||||
p.cost_combo = 0
|
||||
p.cost_diff = p.cost_combo
|
||||
if idx1 == idx2 {
|
||||
return
|
||||
}
|
||||
|
||||
if idx2 < idx1 {
|
||||
var t uint32 = idx2
|
||||
idx2 = idx1
|
||||
idx1 = t
|
||||
}
|
||||
|
||||
p.idx1 = idx1
|
||||
p.idx2 = idx2
|
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
|
||||
p.cost_diff -= out[idx1].bit_cost_
|
||||
p.cost_diff -= out[idx2].bit_cost_
|
||||
|
||||
if out[idx1].total_count_ == 0 {
|
||||
p.cost_combo = out[idx2].bit_cost_
|
||||
is_good_pair = true
|
||||
} else if out[idx2].total_count_ == 0 {
|
||||
p.cost_combo = out[idx1].bit_cost_
|
||||
is_good_pair = true
|
||||
} else {
|
||||
var threshold float64
|
||||
if *num_pairs == 0 {
|
||||
threshold = 1e99
|
||||
} else {
|
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
|
||||
}
|
||||
var combo histogramDistance = out[idx1]
|
||||
var cost_combo float64
|
||||
histogramAddHistogramDistance(&combo, &out[idx2])
|
||||
cost_combo = populationCostDistance(&combo)
|
||||
if cost_combo < threshold-p.cost_diff {
|
||||
p.cost_combo = cost_combo
|
||||
is_good_pair = true
|
||||
}
|
||||
}
|
||||
|
||||
if is_good_pair {
|
||||
p.cost_diff += p.cost_combo
|
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = pairs[0]
|
||||
(*num_pairs)++
|
||||
}
|
||||
|
||||
pairs[0] = p
|
||||
} else if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = p
|
||||
(*num_pairs)++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
|
||||
var cost_diff_threshold float64 = 0.0
|
||||
var min_cluster_size uint = 1
|
||||
var num_pairs uint = 0
|
||||
{
|
||||
/* We maintain a vector of histogram pairs, with the property that the pair
|
||||
with the maximum bit cost reduction is the first. */
|
||||
var idx1 uint
|
||||
for idx1 = 0; idx1 < num_clusters; idx1++ {
|
||||
var idx2 uint
|
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
|
||||
compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for num_clusters > min_cluster_size {
|
||||
var best_idx1 uint32
|
||||
var best_idx2 uint32
|
||||
var i uint
|
||||
if pairs[0].cost_diff >= cost_diff_threshold {
|
||||
cost_diff_threshold = 1e99
|
||||
min_cluster_size = max_clusters
|
||||
continue
|
||||
}
|
||||
|
||||
/* Take the best pair from the top of heap. */
|
||||
best_idx1 = pairs[0].idx1
|
||||
|
||||
best_idx2 = pairs[0].idx2
|
||||
histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
|
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo
|
||||
cluster_size[best_idx1] += cluster_size[best_idx2]
|
||||
for i = 0; i < symbols_size; i++ {
|
||||
if symbols[i] == best_idx2 {
|
||||
symbols[i] = best_idx1
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
if clusters[i] == best_idx2 {
|
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
num_clusters--
|
||||
{
|
||||
/* Remove pairs intersecting the just combined best pair. */
|
||||
var copy_to_idx uint = 0
|
||||
for i = 0; i < num_pairs; i++ {
|
||||
var p *histogramPair = &pairs[i]
|
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
|
||||
/* Remove invalid pair from the queue. */
|
||||
continue
|
||||
}
|
||||
|
||||
if histogramPairIsLess(&pairs[0], p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
var front histogramPair = pairs[0]
|
||||
pairs[0] = *p
|
||||
pairs[copy_to_idx] = front
|
||||
} else {
|
||||
pairs[copy_to_idx] = *p
|
||||
}
|
||||
|
||||
copy_to_idx++
|
||||
}
|
||||
|
||||
num_pairs = copy_to_idx
|
||||
}
|
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
|
||||
return num_clusters
|
||||
}
|
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
|
||||
func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 {
|
||||
if histogram.total_count_ == 0 {
|
||||
return 0.0
|
||||
} else {
|
||||
var tmp histogramDistance = *histogram
|
||||
histogramAddHistogramDistance(&tmp, candidate)
|
||||
return populationCostDistance(&tmp) - candidate.bit_cost_
|
||||
}
|
||||
}
|
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms.
|
||||
When called, clusters[0..num_clusters) contains the unique values from
|
||||
symbols[0..in_size), but this property is not preserved in this function.
|
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */
|
||||
func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) {
|
||||
var i uint
|
||||
for i = 0; i < in_size; i++ {
|
||||
var best_out uint32
|
||||
if i == 0 {
|
||||
best_out = symbols[0]
|
||||
} else {
|
||||
best_out = symbols[i-1]
|
||||
}
|
||||
var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out])
|
||||
var j uint
|
||||
for j = 0; j < num_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
symbols[i] = best_out
|
||||
}
|
||||
|
||||
/* Recompute each out based on raw and symbols. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
histogramClearDistance(&out[clusters[i]])
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
histogramAddHistogramDistance(&out[symbols[i]], &in[i])
|
||||
}
|
||||
}
|
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in
|
||||
symbols[0..length) array in the following way:
|
||||
* when called, symbols[] contains indexes into out[], and has N unique
|
||||
values (possibly N < length)
|
||||
* on return, symbols'[i] = f(symbols[i]) and
|
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
|
||||
where f is a bijection between the range of symbols[] and [0..N), and
|
||||
the first occurrences of values in symbols'[i] come in consecutive
|
||||
increasing order.
|
||||
Returns N, the number of unique values in symbols[]. */
|
||||
|
||||
var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint {
|
||||
var new_index []uint32 = make([]uint32, length)
|
||||
var next_index uint32
|
||||
var tmp []histogramDistance
|
||||
var i uint
|
||||
for i = 0; i < length; i++ {
|
||||
new_index[i] = histogramReindexDistance_kInvalidIndex
|
||||
}
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex {
|
||||
new_index[symbols[i]] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
||||
tmp and reduce the number of copying by the factor of 2. */
|
||||
tmp = make([]histogramDistance, next_index)
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == next_index {
|
||||
tmp[next_index] = out[symbols[i]]
|
||||
next_index++
|
||||
}
|
||||
|
||||
symbols[i] = new_index[symbols[i]]
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
for i = 0; uint32(i) < next_index; i++ {
|
||||
out[i] = tmp[i]
|
||||
}
|
||||
|
||||
tmp = nil
|
||||
return uint(next_index)
|
||||
}
|
||||
|
||||
func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) {
|
||||
var cluster_size []uint32 = make([]uint32, in_size)
|
||||
var clusters []uint32 = make([]uint32, in_size)
|
||||
var num_clusters uint = 0
|
||||
var max_input_histograms uint = 64
|
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
|
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
|
||||
var i uint
|
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */
|
||||
for i = 0; i < in_size; i++ {
|
||||
cluster_size[i] = 1
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
out[i] = in[i]
|
||||
out[i].bit_cost_ = populationCostDistance(&in[i])
|
||||
histogram_symbols[i] = uint32(i)
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms {
|
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
clusters[num_clusters+j] = uint32(i + j)
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
|
||||
num_clusters += num_new_clusters
|
||||
}
|
||||
{
|
||||
/* For the second pass, we limit the total number of histogram pairs.
|
||||
After this limit is reached, we only keep searching for the best pair. */
|
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < (max_num_pairs + 1) {
|
||||
var _new_size uint
|
||||
if pairs_capacity == 0 {
|
||||
_new_size = max_num_pairs + 1
|
||||
} else {
|
||||
_new_size = pairs_capacity
|
||||
}
|
||||
var new_array []histogramPair
|
||||
for _new_size < (max_num_pairs + 1) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramPair, _new_size)
|
||||
if pairs_capacity != 0 {
|
||||
copy(new_array, pairs[:pairs_capacity])
|
||||
}
|
||||
|
||||
pairs = new_array
|
||||
pairs_capacity = _new_size
|
||||
}
|
||||
|
||||
/* Collapse similar histograms. */
|
||||
num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
|
||||
}
|
||||
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */
|
||||
histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols)
|
||||
|
||||
clusters = nil
|
||||
|
||||
/* Convert the context map to a canonical form. */
|
||||
*out_size = histogramReindexDistance(out, histogram_symbols, in_size)
|
||||
}
|
||||
326
vendor/github.com/andybalholm/brotli/cluster_literal.go
generated
vendored
Normal file
326
vendor/github.com/andybalholm/brotli/cluster_literal.go
generated
vendored
Normal file
@ -0,0 +1,326 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
|
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
|
||||
func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
|
||||
var is_good_pair bool = false
|
||||
var p histogramPair
|
||||
p.idx2 = 0
|
||||
p.idx1 = p.idx2
|
||||
p.cost_combo = 0
|
||||
p.cost_diff = p.cost_combo
|
||||
if idx1 == idx2 {
|
||||
return
|
||||
}
|
||||
|
||||
if idx2 < idx1 {
|
||||
var t uint32 = idx2
|
||||
idx2 = idx1
|
||||
idx1 = t
|
||||
}
|
||||
|
||||
p.idx1 = idx1
|
||||
p.idx2 = idx2
|
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
|
||||
p.cost_diff -= out[idx1].bit_cost_
|
||||
p.cost_diff -= out[idx2].bit_cost_
|
||||
|
||||
if out[idx1].total_count_ == 0 {
|
||||
p.cost_combo = out[idx2].bit_cost_
|
||||
is_good_pair = true
|
||||
} else if out[idx2].total_count_ == 0 {
|
||||
p.cost_combo = out[idx1].bit_cost_
|
||||
is_good_pair = true
|
||||
} else {
|
||||
var threshold float64
|
||||
if *num_pairs == 0 {
|
||||
threshold = 1e99
|
||||
} else {
|
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
|
||||
}
|
||||
var combo histogramLiteral = out[idx1]
|
||||
var cost_combo float64
|
||||
histogramAddHistogramLiteral(&combo, &out[idx2])
|
||||
cost_combo = populationCostLiteral(&combo)
|
||||
if cost_combo < threshold-p.cost_diff {
|
||||
p.cost_combo = cost_combo
|
||||
is_good_pair = true
|
||||
}
|
||||
}
|
||||
|
||||
if is_good_pair {
|
||||
p.cost_diff += p.cost_combo
|
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = pairs[0]
|
||||
(*num_pairs)++
|
||||
}
|
||||
|
||||
pairs[0] = p
|
||||
} else if *num_pairs < max_num_pairs {
|
||||
pairs[*num_pairs] = p
|
||||
(*num_pairs)++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
|
||||
var cost_diff_threshold float64 = 0.0
|
||||
var min_cluster_size uint = 1
|
||||
var num_pairs uint = 0
|
||||
{
|
||||
/* We maintain a vector of histogram pairs, with the property that the pair
|
||||
with the maximum bit cost reduction is the first. */
|
||||
var idx1 uint
|
||||
for idx1 = 0; idx1 < num_clusters; idx1++ {
|
||||
var idx2 uint
|
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
|
||||
compareAndPushToQueueLiteral(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for num_clusters > min_cluster_size {
|
||||
var best_idx1 uint32
|
||||
var best_idx2 uint32
|
||||
var i uint
|
||||
if pairs[0].cost_diff >= cost_diff_threshold {
|
||||
cost_diff_threshold = 1e99
|
||||
min_cluster_size = max_clusters
|
||||
continue
|
||||
}
|
||||
|
||||
/* Take the best pair from the top of heap. */
|
||||
best_idx1 = pairs[0].idx1
|
||||
|
||||
best_idx2 = pairs[0].idx2
|
||||
histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2])
|
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo
|
||||
cluster_size[best_idx1] += cluster_size[best_idx2]
|
||||
for i = 0; i < symbols_size; i++ {
|
||||
if symbols[i] == best_idx2 {
|
||||
symbols[i] = best_idx1
|
||||
}
|
||||
}
|
||||
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
if clusters[i] == best_idx2 {
|
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
num_clusters--
|
||||
{
|
||||
/* Remove pairs intersecting the just combined best pair. */
|
||||
var copy_to_idx uint = 0
|
||||
for i = 0; i < num_pairs; i++ {
|
||||
var p *histogramPair = &pairs[i]
|
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
|
||||
/* Remove invalid pair from the queue. */
|
||||
continue
|
||||
}
|
||||
|
||||
if histogramPairIsLess(&pairs[0], p) {
|
||||
/* Replace the top of the queue if needed. */
|
||||
var front histogramPair = pairs[0]
|
||||
pairs[0] = *p
|
||||
pairs[copy_to_idx] = front
|
||||
} else {
|
||||
pairs[copy_to_idx] = *p
|
||||
}
|
||||
|
||||
copy_to_idx++
|
||||
}
|
||||
|
||||
num_pairs = copy_to_idx
|
||||
}
|
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
compareAndPushToQueueLiteral(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
|
||||
}
|
||||
}
|
||||
|
||||
return num_clusters
|
||||
}
|
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
|
||||
func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 {
|
||||
if histogram.total_count_ == 0 {
|
||||
return 0.0
|
||||
} else {
|
||||
var tmp histogramLiteral = *histogram
|
||||
histogramAddHistogramLiteral(&tmp, candidate)
|
||||
return populationCostLiteral(&tmp) - candidate.bit_cost_
|
||||
}
|
||||
}
|
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms.
|
||||
When called, clusters[0..num_clusters) contains the unique values from
|
||||
symbols[0..in_size), but this property is not preserved in this function.
|
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */
|
||||
func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) {
|
||||
var i uint
|
||||
for i = 0; i < in_size; i++ {
|
||||
var best_out uint32
|
||||
if i == 0 {
|
||||
best_out = symbols[0]
|
||||
} else {
|
||||
best_out = symbols[i-1]
|
||||
}
|
||||
var best_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[best_out])
|
||||
var j uint
|
||||
for j = 0; j < num_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
symbols[i] = best_out
|
||||
}
|
||||
|
||||
/* Recompute each out based on raw and symbols. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
histogramClearLiteral(&out[clusters[i]])
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
histogramAddHistogramLiteral(&out[symbols[i]], &in[i])
|
||||
}
|
||||
}
|
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in
|
||||
symbols[0..length) array in the following way:
|
||||
* when called, symbols[] contains indexes into out[], and has N unique
|
||||
values (possibly N < length)
|
||||
* on return, symbols'[i] = f(symbols[i]) and
|
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
|
||||
where f is a bijection between the range of symbols[] and [0..N), and
|
||||
the first occurrences of values in symbols'[i] come in consecutive
|
||||
increasing order.
|
||||
Returns N, the number of unique values in symbols[]. */
|
||||
|
||||
var histogramReindexLiteral_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint {
|
||||
var new_index []uint32 = make([]uint32, length)
|
||||
var next_index uint32
|
||||
var tmp []histogramLiteral
|
||||
var i uint
|
||||
for i = 0; i < length; i++ {
|
||||
new_index[i] = histogramReindexLiteral_kInvalidIndex
|
||||
}
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == histogramReindexLiteral_kInvalidIndex {
|
||||
new_index[symbols[i]] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
||||
tmp and reduce the number of copying by the factor of 2. */
|
||||
tmp = make([]histogramLiteral, next_index)
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == next_index {
|
||||
tmp[next_index] = out[symbols[i]]
|
||||
next_index++
|
||||
}
|
||||
|
||||
symbols[i] = new_index[symbols[i]]
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
for i = 0; uint32(i) < next_index; i++ {
|
||||
out[i] = tmp[i]
|
||||
}
|
||||
|
||||
tmp = nil
|
||||
return uint(next_index)
|
||||
}
|
||||
|
||||
func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) {
|
||||
var cluster_size []uint32 = make([]uint32, in_size)
|
||||
var clusters []uint32 = make([]uint32, in_size)
|
||||
var num_clusters uint = 0
|
||||
var max_input_histograms uint = 64
|
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
|
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
|
||||
var i uint
|
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */
|
||||
for i = 0; i < in_size; i++ {
|
||||
cluster_size[i] = 1
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
out[i] = in[i]
|
||||
out[i].bit_cost_ = populationCostLiteral(&in[i])
|
||||
histogram_symbols[i] = uint32(i)
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms {
|
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
clusters[num_clusters+j] = uint32(i + j)
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
|
||||
num_clusters += num_new_clusters
|
||||
}
|
||||
{
|
||||
/* For the second pass, we limit the total number of histogram pairs.
|
||||
After this limit is reached, we only keep searching for the best pair. */
|
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < (max_num_pairs + 1) {
|
||||
var _new_size uint
|
||||
if pairs_capacity == 0 {
|
||||
_new_size = max_num_pairs + 1
|
||||
} else {
|
||||
_new_size = pairs_capacity
|
||||
}
|
||||
var new_array []histogramPair
|
||||
for _new_size < (max_num_pairs + 1) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramPair, _new_size)
|
||||
if pairs_capacity != 0 {
|
||||
copy(new_array, pairs[:pairs_capacity])
|
||||
}
|
||||
|
||||
pairs = new_array
|
||||
pairs_capacity = _new_size
|
||||
}
|
||||
|
||||
/* Collapse similar histograms. */
|
||||
num_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
|
||||
}
|
||||
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */
|
||||
histogramRemapLiteral(in, in_size, clusters, num_clusters, out, histogram_symbols)
|
||||
|
||||
clusters = nil
|
||||
|
||||
/* Convert the context map to a canonical form. */
|
||||
*out_size = histogramReindexLiteral(out, histogram_symbols, in_size)
|
||||
}
|
||||
254
vendor/github.com/andybalholm/brotli/command.go
generated
vendored
Normal file
254
vendor/github.com/andybalholm/brotli/command.go
generated
vendored
Normal file
@ -0,0 +1,254 @@
|
||||
package brotli
|
||||
|
||||
var kInsBase = []uint32{
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
8,
|
||||
10,
|
||||
14,
|
||||
18,
|
||||
26,
|
||||
34,
|
||||
50,
|
||||
66,
|
||||
98,
|
||||
130,
|
||||
194,
|
||||
322,
|
||||
578,
|
||||
1090,
|
||||
2114,
|
||||
6210,
|
||||
22594,
|
||||
}
|
||||
|
||||
var kInsExtra = []uint32{
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
5,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
12,
|
||||
14,
|
||||
24,
|
||||
}
|
||||
|
||||
var kCopyBase = []uint32{
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
12,
|
||||
14,
|
||||
18,
|
||||
22,
|
||||
30,
|
||||
38,
|
||||
54,
|
||||
70,
|
||||
102,
|
||||
134,
|
||||
198,
|
||||
326,
|
||||
582,
|
||||
1094,
|
||||
2118,
|
||||
}
|
||||
|
||||
var kCopyExtra = []uint32{
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
5,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
24,
|
||||
}
|
||||
|
||||
func getInsertLengthCode(insertlen uint) uint16 {
|
||||
if insertlen < 6 {
|
||||
return uint16(insertlen)
|
||||
} else if insertlen < 130 {
|
||||
var nbits uint32 = log2FloorNonZero(insertlen-2) - 1
|
||||
return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2)
|
||||
} else if insertlen < 2114 {
|
||||
return uint16(log2FloorNonZero(insertlen-66) + 10)
|
||||
} else if insertlen < 6210 {
|
||||
return 21
|
||||
} else if insertlen < 22594 {
|
||||
return 22
|
||||
} else {
|
||||
return 23
|
||||
}
|
||||
}
|
||||
|
||||
func getCopyLengthCode(copylen uint) uint16 {
|
||||
if copylen < 10 {
|
||||
return uint16(copylen - 2)
|
||||
} else if copylen < 134 {
|
||||
var nbits uint32 = log2FloorNonZero(copylen-6) - 1
|
||||
return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4)
|
||||
} else if copylen < 2118 {
|
||||
return uint16(log2FloorNonZero(copylen-70) + 12)
|
||||
} else {
|
||||
return 23
|
||||
}
|
||||
}
|
||||
|
||||
func combineLengthCodes(inscode uint16, copycode uint16, use_last_distance bool) uint16 {
|
||||
var bits64 uint16 = uint16(copycode&0x7 | (inscode&0x7)<<3)
|
||||
if use_last_distance && inscode < 8 && copycode < 16 {
|
||||
if copycode < 8 {
|
||||
return bits64
|
||||
} else {
|
||||
return bits64 | 64
|
||||
}
|
||||
} else {
|
||||
/* Specification: 5 Encoding of ... (last table) */
|
||||
/* offset = 2 * index, where index is in range [0..8] */
|
||||
var offset uint32 = 2 * ((uint32(copycode) >> 3) + 3*(uint32(inscode)>>3))
|
||||
|
||||
/* All values in specification are K * 64,
|
||||
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
|
||||
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
||||
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
|
||||
All values in D require only 2 bits to encode.
|
||||
Magic constant is shifted 6 bits left, to avoid final multiplication. */
|
||||
offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0)
|
||||
|
||||
return uint16(offset | uint32(bits64))
|
||||
}
|
||||
}
|
||||
|
||||
func getLengthCode(insertlen uint, copylen uint, use_last_distance bool, code *uint16) {
|
||||
var inscode uint16 = getInsertLengthCode(insertlen)
|
||||
var copycode uint16 = getCopyLengthCode(copylen)
|
||||
*code = combineLengthCodes(inscode, copycode, use_last_distance)
|
||||
}
|
||||
|
||||
func getInsertBase(inscode uint16) uint32 {
|
||||
return kInsBase[inscode]
|
||||
}
|
||||
|
||||
func getInsertExtra(inscode uint16) uint32 {
|
||||
return kInsExtra[inscode]
|
||||
}
|
||||
|
||||
func getCopyBase(copycode uint16) uint32 {
|
||||
return kCopyBase[copycode]
|
||||
}
|
||||
|
||||
func getCopyExtra(copycode uint16) uint32 {
|
||||
return kCopyExtra[copycode]
|
||||
}
|
||||
|
||||
type command struct {
|
||||
insert_len_ uint32
|
||||
copy_len_ uint32
|
||||
dist_extra_ uint32
|
||||
cmd_prefix_ uint16
|
||||
dist_prefix_ uint16
|
||||
}
|
||||
|
||||
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
||||
func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) {
|
||||
/* Don't rely on signed int representation, use honest casts. */
|
||||
var delta uint32 = uint32(byte(int8(copylen_code_delta)))
|
||||
cmd.insert_len_ = uint32(insertlen)
|
||||
cmd.copy_len_ = uint32(uint32(copylen) | delta<<25)
|
||||
|
||||
/* The distance prefix and extra bits are stored in this Command as if
|
||||
npostfix and ndirect were 0, they are only recomputed later after the
|
||||
clustering if needed. */
|
||||
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
|
||||
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func makeInsertCommand(insertlen uint) (cmd command) {
|
||||
cmd.insert_len_ = uint32(insertlen)
|
||||
cmd.copy_len_ = 4 << 25
|
||||
cmd.dist_extra_ = 0
|
||||
cmd.dist_prefix_ = numDistanceShortCodes
|
||||
getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
|
||||
if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes {
|
||||
return uint32(self.dist_prefix_) & 0x3FF
|
||||
} else {
|
||||
var dcode uint32 = uint32(self.dist_prefix_) & 0x3FF
|
||||
var nbits uint32 = uint32(self.dist_prefix_) >> 10
|
||||
var extra uint32 = self.dist_extra_
|
||||
var postfix_mask uint32 = (1 << dist.distance_postfix_bits) - 1
|
||||
var hcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) >> dist.distance_postfix_bits
|
||||
var lcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) & postfix_mask
|
||||
var offset uint32 = ((2 + (hcode & 1)) << nbits) - 4
|
||||
return ((offset + extra) << dist.distance_postfix_bits) + lcode + dist.num_direct_distance_codes + numDistanceShortCodes
|
||||
}
|
||||
}
|
||||
|
||||
func commandDistanceContext(self *command) uint32 {
|
||||
var r uint32 = uint32(self.cmd_prefix_) >> 6
|
||||
var c uint32 = uint32(self.cmd_prefix_) & 7
|
||||
if (r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2) {
|
||||
return c
|
||||
}
|
||||
|
||||
return 3
|
||||
}
|
||||
|
||||
func commandCopyLen(self *command) uint32 {
|
||||
return self.copy_len_ & 0x1FFFFFF
|
||||
}
|
||||
|
||||
func commandCopyLenCode(self *command) uint32 {
|
||||
var modifier uint32 = self.copy_len_ >> 25
|
||||
var delta int32 = int32(int8(byte(modifier | (modifier&0x40)<<1)))
|
||||
return uint32(int32(self.copy_len_&0x1FFFFFF) + delta)
|
||||
}
|
||||
834
vendor/github.com/andybalholm/brotli/compress_fragment.go
generated
vendored
Normal file
834
vendor/github.com/andybalholm/brotli/compress_fragment.go
generated
vendored
Normal file
@ -0,0 +1,834 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input
|
||||
history. This function uses one-pass processing: when we find a backward
|
||||
match, we immediately emit the corresponding command and literal codes to
|
||||
the bit stream.
|
||||
|
||||
Adapted from the CompressFragment() function in
|
||||
https://github.com/google/snappy/blob/master/snappy.cc */
|
||||
|
||||
const maxDistance_compress_fragment = 262128
|
||||
|
||||
func hash5(p []byte, shift uint) uint32 {
|
||||
var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32)
|
||||
return uint32(h >> shift)
|
||||
}
|
||||
|
||||
func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 {
|
||||
assert(offset >= 0)
|
||||
assert(offset <= 3)
|
||||
{
|
||||
var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32)
|
||||
return uint32(h >> shift)
|
||||
}
|
||||
}
|
||||
|
||||
func isMatch5(p1 []byte, p2 []byte) bool {
|
||||
return binary.LittleEndian.Uint32(p1) == binary.LittleEndian.Uint32(p2) &&
|
||||
p1[4] == p2[4]
|
||||
}
|
||||
|
||||
/* Builds a literal prefix code into "depths" and "bits" based on the statistics
|
||||
of the "input" string and stores it into the bit stream.
|
||||
Note that the prefix code here is built from the pre-LZ77 input, therefore
|
||||
we can only approximate the statistics of the actual literal stream.
|
||||
Moreover, for long inputs we build a histogram from a sample of the input
|
||||
and thus have to assign a non-zero depth for each literal.
|
||||
Returns estimated compression ratio millibytes/char for encoding given input
|
||||
with generated code. */
|
||||
func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint {
|
||||
var histogram = [256]uint32{0}
|
||||
var histogram_total uint
|
||||
var i uint
|
||||
if input_size < 1<<15 {
|
||||
for i = 0; i < input_size; i++ {
|
||||
histogram[input[i]]++
|
||||
}
|
||||
|
||||
histogram_total = input_size
|
||||
for i = 0; i < 256; i++ {
|
||||
/* We weigh the first 11 samples with weight 3 to account for the
|
||||
balancing effect of the LZ77 phase on the histogram. */
|
||||
var adjust uint32 = 2 * brotli_min_uint32_t(histogram[i], 11)
|
||||
histogram[i] += adjust
|
||||
histogram_total += uint(adjust)
|
||||
}
|
||||
} else {
|
||||
const kSampleRate uint = 29
|
||||
for i = 0; i < input_size; i += kSampleRate {
|
||||
histogram[input[i]]++
|
||||
}
|
||||
|
||||
histogram_total = (input_size + kSampleRate - 1) / kSampleRate
|
||||
for i = 0; i < 256; i++ {
|
||||
/* We add 1 to each population count to avoid 0 bit depths (since this is
|
||||
only a sample and we don't know if the symbol appears or not), and we
|
||||
weigh the first 11 samples with weight 3 to account for the balancing
|
||||
effect of the LZ77 phase on the histogram (more frequent symbols are
|
||||
more likely to be in backward references instead as literals). */
|
||||
var adjust uint32 = 1 + 2*brotli_min_uint32_t(histogram[i], 11)
|
||||
histogram[i] += adjust
|
||||
histogram_total += uint(adjust)
|
||||
}
|
||||
}
|
||||
|
||||
buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */
|
||||
8, depths, bits, storage_ix, storage)
|
||||
{
|
||||
var literal_ratio uint = 0
|
||||
for i = 0; i < 256; i++ {
|
||||
if histogram[i] != 0 {
|
||||
literal_ratio += uint(histogram[i] * uint32(depths[i]))
|
||||
}
|
||||
}
|
||||
|
||||
/* Estimated encoding ratio, millibytes per symbol. */
|
||||
return (literal_ratio * 125) / histogram_total
|
||||
}
|
||||
}
|
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
|
||||
"bits" based on "histogram" and stores it into the bit stream. */
|
||||
func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
|
||||
var tree [129]huffmanTree
|
||||
var cmd_depth = [numCommandSymbols]byte{0}
|
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
||||
|
||||
var cmd_bits [64]uint16
|
||||
|
||||
createHuffmanTree(histogram, 64, 15, tree[:], depth)
|
||||
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
|
||||
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
functions. */
|
||||
copy(cmd_depth[:], depth[:24])
|
||||
|
||||
copy(cmd_depth[24:][:], depth[40:][:8])
|
||||
copy(cmd_depth[32:][:], depth[24:][:8])
|
||||
copy(cmd_depth[40:][:], depth[48:][:8])
|
||||
copy(cmd_depth[48:][:], depth[32:][:8])
|
||||
copy(cmd_depth[56:][:], depth[56:][:8])
|
||||
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
|
||||
copy(bits, cmd_bits[:24])
|
||||
copy(bits[24:], cmd_bits[32:][:8])
|
||||
copy(bits[32:], cmd_bits[48:][:8])
|
||||
copy(bits[40:], cmd_bits[24:][:8])
|
||||
copy(bits[48:], cmd_bits[40:][:8])
|
||||
copy(bits[56:], cmd_bits[56:][:8])
|
||||
convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
|
||||
{
|
||||
/* Create the bit length array for the full command alphabet. */
|
||||
var i uint
|
||||
for i := 0; i < int(64); i++ {
|
||||
cmd_depth[i] = 0
|
||||
} /* only 64 first values were used */
|
||||
copy(cmd_depth[:], depth[:8])
|
||||
copy(cmd_depth[64:][:], depth[8:][:8])
|
||||
copy(cmd_depth[128:][:], depth[16:][:8])
|
||||
copy(cmd_depth[192:][:], depth[24:][:8])
|
||||
copy(cmd_depth[384:][:], depth[32:][:8])
|
||||
for i = 0; i < 8; i++ {
|
||||
cmd_depth[128+8*i] = depth[40+i]
|
||||
cmd_depth[256+8*i] = depth[48+i]
|
||||
cmd_depth[448+8*i] = depth[56+i]
|
||||
}
|
||||
|
||||
storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
|
||||
}
|
||||
|
||||
storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
|
||||
}
|
||||
|
||||
/* REQUIRES: insertlen < 6210 */
|
||||
func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
|
||||
if insertlen < 6 {
|
||||
var code uint = insertlen + 40
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
histo[code]++
|
||||
} else if insertlen < 130 {
|
||||
var tail uint = insertlen - 2
|
||||
var nbits uint32 = log2FloorNonZero(tail) - 1
|
||||
var prefix uint = tail >> nbits
|
||||
var inscode uint = uint((nbits << 1) + uint32(prefix) + 42)
|
||||
writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
|
||||
histo[inscode]++
|
||||
} else if insertlen < 2114 {
|
||||
var tail uint = insertlen - 66
|
||||
var nbits uint32 = log2FloorNonZero(tail)
|
||||
var code uint = uint(nbits + 50)
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
|
||||
histo[code]++
|
||||
} else {
|
||||
writeBits(uint(depth[61]), uint64(bits[61]), storage_ix, storage)
|
||||
writeBits(12, uint64(insertlen)-2114, storage_ix, storage)
|
||||
histo[61]++
|
||||
}
|
||||
}
|
||||
|
||||
func emitLongInsertLen(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
|
||||
if insertlen < 22594 {
|
||||
writeBits(uint(depth[62]), uint64(bits[62]), storage_ix, storage)
|
||||
writeBits(14, uint64(insertlen)-6210, storage_ix, storage)
|
||||
histo[62]++
|
||||
} else {
|
||||
writeBits(uint(depth[63]), uint64(bits[63]), storage_ix, storage)
|
||||
writeBits(24, uint64(insertlen)-22594, storage_ix, storage)
|
||||
histo[63]++
|
||||
}
|
||||
}
|
||||
|
||||
func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
|
||||
if copylen < 10 {
|
||||
writeBits(uint(depth[copylen+14]), uint64(bits[copylen+14]), storage_ix, storage)
|
||||
histo[copylen+14]++
|
||||
} else if copylen < 134 {
|
||||
var tail uint = copylen - 6
|
||||
var nbits uint32 = log2FloorNonZero(tail) - 1
|
||||
var prefix uint = tail >> nbits
|
||||
var code uint = uint((nbits << 1) + uint32(prefix) + 20)
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
|
||||
histo[code]++
|
||||
} else if copylen < 2118 {
|
||||
var tail uint = copylen - 70
|
||||
var nbits uint32 = log2FloorNonZero(tail)
|
||||
var code uint = uint(nbits + 28)
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
|
||||
histo[code]++
|
||||
} else {
|
||||
writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
|
||||
writeBits(24, uint64(copylen)-2118, storage_ix, storage)
|
||||
histo[39]++
|
||||
}
|
||||
}
|
||||
|
||||
func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
|
||||
if copylen < 12 {
|
||||
writeBits(uint(depth[copylen-4]), uint64(bits[copylen-4]), storage_ix, storage)
|
||||
histo[copylen-4]++
|
||||
} else if copylen < 72 {
|
||||
var tail uint = copylen - 8
|
||||
var nbits uint32 = log2FloorNonZero(tail) - 1
|
||||
var prefix uint = tail >> nbits
|
||||
var code uint = uint((nbits << 1) + uint32(prefix) + 4)
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
|
||||
histo[code]++
|
||||
} else if copylen < 136 {
|
||||
var tail uint = copylen - 8
|
||||
var code uint = (tail >> 5) + 30
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(5, uint64(tail)&31, storage_ix, storage)
|
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
|
||||
histo[code]++
|
||||
histo[64]++
|
||||
} else if copylen < 2120 {
|
||||
var tail uint = copylen - 72
|
||||
var nbits uint32 = log2FloorNonZero(tail)
|
||||
var code uint = uint(nbits + 28)
|
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
|
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
|
||||
histo[code]++
|
||||
histo[64]++
|
||||
} else {
|
||||
writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
|
||||
writeBits(24, uint64(copylen)-2120, storage_ix, storage)
|
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
|
||||
histo[39]++
|
||||
histo[64]++
|
||||
}
|
||||
}
|
||||
|
||||
func emitDistance1(distance uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
|
||||
var d uint = distance + 3
|
||||
var nbits uint32 = log2FloorNonZero(d) - 1
|
||||
var prefix uint = (d >> nbits) & 1
|
||||
var offset uint = (2 + prefix) << nbits
|
||||
var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80)
|
||||
writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage)
|
||||
writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage)
|
||||
histo[distcode]++
|
||||
}
|
||||
|
||||
func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
|
||||
var j uint
|
||||
for j = 0; j < len; j++ {
|
||||
var lit byte = input[j]
|
||||
writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage)
|
||||
}
|
||||
}
|
||||
|
||||
/* REQUIRES: len <= 1 << 24. */
|
||||
func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
|
||||
var nibbles uint = 6
|
||||
|
||||
/* ISLAST */
|
||||
writeBits(1, 0, storage_ix, storage)
|
||||
|
||||
if len <= 1<<16 {
|
||||
nibbles = 4
|
||||
} else if len <= 1<<20 {
|
||||
nibbles = 5
|
||||
}
|
||||
|
||||
writeBits(2, uint64(nibbles)-4, storage_ix, storage)
|
||||
writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
|
||||
|
||||
/* ISUNCOMPRESSED */
|
||||
writeSingleBit(is_uncompressed, storage_ix, storage)
|
||||
}
|
||||
|
||||
func updateBits(n_bits uint, bits uint32, pos uint, array []byte) {
|
||||
for n_bits > 0 {
|
||||
var byte_pos uint = pos >> 3
|
||||
var n_unchanged_bits uint = pos & 7
|
||||
var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits)
|
||||
var total_bits uint = n_unchanged_bits + n_changed_bits
|
||||
var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1)
|
||||
var unchanged_bits uint32 = uint32(array[byte_pos]) & mask
|
||||
var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1)
|
||||
array[byte_pos] = byte(changed_bits<<n_unchanged_bits | unchanged_bits)
|
||||
n_bits -= n_changed_bits
|
||||
bits >>= n_changed_bits
|
||||
pos += n_changed_bits
|
||||
}
|
||||
}
|
||||
|
||||
func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) {
|
||||
var bitpos uint = new_storage_ix & 7
|
||||
var mask uint = (1 << bitpos) - 1
|
||||
storage[new_storage_ix>>3] &= byte(mask)
|
||||
*storage_ix = new_storage_ix
|
||||
}
|
||||
|
||||
var shouldMergeBlock_kSampleRate uint = 43
|
||||
|
||||
func shouldMergeBlock(data []byte, len uint, depths []byte) bool {
|
||||
var histo = [256]uint{0}
|
||||
var i uint
|
||||
for i = 0; i < len; i += shouldMergeBlock_kSampleRate {
|
||||
histo[data[i]]++
|
||||
}
|
||||
{
|
||||
var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate
|
||||
var r float64 = (fastLog2(total)+0.5)*float64(total) + 200
|
||||
for i = 0; i < 256; i++ {
|
||||
r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i]))
|
||||
}
|
||||
|
||||
return r >= 0.0
|
||||
}
|
||||
}
|
||||
|
||||
func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertlen uint, literal_ratio uint) bool {
|
||||
var compressed uint = uint(-cap(next_emit) + cap(metablock_start))
|
||||
if compressed*50 > insertlen {
|
||||
return false
|
||||
} else {
|
||||
return literal_ratio > 980
|
||||
}
|
||||
}
|
||||
|
||||
func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) {
|
||||
var len uint = uint(-cap(end) + cap(begin))
|
||||
rewindBitPosition1(storage_ix_start, storage_ix, storage)
|
||||
storeMetaBlockHeader1(uint(len), true, storage_ix, storage)
|
||||
*storage_ix = (*storage_ix + 7) &^ 7
|
||||
copy(storage[*storage_ix>>3:], begin[:len])
|
||||
*storage_ix += uint(len << 3)
|
||||
storage[*storage_ix>>3] = 0
|
||||
}
|
||||
|
||||
var kCmdHistoSeed = [128]uint32{
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
}
|
||||
|
||||
var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15
|
||||
var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16
|
||||
|
||||
func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
|
||||
var cmd_histo [128]uint32
|
||||
var ip_end int
|
||||
var next_emit int = 0
|
||||
var base_ip int = 0
|
||||
var input int = 0
|
||||
const kInputMarginBytes uint = windowGap
|
||||
const kMinMatchLen uint = 5
|
||||
var metablock_start int = input
|
||||
var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
|
||||
var total_block_size uint = block_size
|
||||
var mlen_storage_ix uint = *storage_ix + 3
|
||||
var lit_depth [256]byte
|
||||
var lit_bits [256]uint16
|
||||
var literal_ratio uint
|
||||
var ip int
|
||||
var last_distance int
|
||||
var shift uint = 64 - table_bits
|
||||
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a
|
||||
previous copy. Bytes between "next_emit" and the start of the next copy or
|
||||
the end of the input will be emitted as literal bytes. */
|
||||
|
||||
/* Save the start of the first block for position and distance computations.
|
||||
*/
|
||||
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that
|
||||
we can update it later if we decide to extend this meta-block. */
|
||||
storeMetaBlockHeader1(block_size, false, storage_ix, storage)
|
||||
|
||||
/* No block splits, no contexts. */
|
||||
writeBits(13, 0, storage_ix, storage)
|
||||
|
||||
literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||
{
|
||||
/* Store the pre-compressed command and distance prefix codes. */
|
||||
var i uint
|
||||
for i = 0; i+7 < *cmd_code_numbits; i += 8 {
|
||||
writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage)
|
||||
}
|
||||
}
|
||||
|
||||
writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage)
|
||||
|
||||
/* Initialize the command and distance histograms. We will gather
|
||||
statistics of command and distance codes during the processing
|
||||
of this block and use it to update the command and distance
|
||||
prefix codes for the next block. */
|
||||
emit_commands:
|
||||
copy(cmd_histo[:], kCmdHistoSeed[:])
|
||||
|
||||
/* "ip" is the input pointer. */
|
||||
ip = input
|
||||
|
||||
last_distance = -1
|
||||
ip_end = int(uint(input) + block_size)
|
||||
|
||||
if block_size >= kInputMarginBytes {
|
||||
var len_limit uint = brotli_min_size_t(block_size-kMinMatchLen, input_size-kInputMarginBytes)
|
||||
var ip_limit int = int(uint(input) + len_limit)
|
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be
|
||||
sure that all distances are at most window size - 16.
|
||||
For all other blocks, we only need to keep a margin of 5 bytes so that
|
||||
we don't go over the block size with a copy. */
|
||||
|
||||
var next_hash uint32
|
||||
ip++
|
||||
for next_hash = hash5(in[ip:], shift); ; {
|
||||
var skip uint32 = 32
|
||||
var next_ip int = ip
|
||||
/* Step 1: Scan forward in the input looking for a 5-byte-long match.
|
||||
If we get close to exhausting the input then goto emit_remainder.
|
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches
|
||||
found, start looking only at every other byte. If 32 more bytes are
|
||||
scanned, look at every third byte, etc.. When a match is found,
|
||||
immediately go back to looking at every byte. This is a small loss
|
||||
(~5% performance, ~0.1% density) for compressible data due to more
|
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
|
||||
win since the compressor quickly "realizes" the data is incompressible
|
||||
and doesn't bother looking for matches everywhere.
|
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the
|
||||
last match; dividing it by 32 (i.e. right-shifting by five) gives the
|
||||
number of bytes to move ahead for each iteration. */
|
||||
|
||||
var candidate int
|
||||
assert(next_emit < ip)
|
||||
|
||||
trawl:
|
||||
for {
|
||||
var hash uint32 = next_hash
|
||||
var bytes_between_hash_lookups uint32 = skip >> 5
|
||||
skip++
|
||||
assert(hash == hash5(in[next_ip:], shift))
|
||||
ip = next_ip
|
||||
next_ip = int(uint32(ip) + bytes_between_hash_lookups)
|
||||
if next_ip > ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
|
||||
next_hash = hash5(in[next_ip:], shift)
|
||||
candidate = ip - last_distance
|
||||
if isMatch5(in[ip:], in[candidate:]) {
|
||||
if candidate < ip {
|
||||
table[hash] = int(ip - base_ip)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
candidate = base_ip + table[hash]
|
||||
assert(candidate >= base_ip)
|
||||
assert(candidate < ip)
|
||||
|
||||
table[hash] = int(ip - base_ip)
|
||||
if isMatch5(in[ip:], in[candidate:]) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search.
|
||||
Checking is done outside of hot loop to reduce overhead. */
|
||||
if ip-candidate > maxDistance_compress_fragment {
|
||||
goto trawl
|
||||
}
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit" to the bit stream, and then see if we can find a next match
|
||||
immediately afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
{
|
||||
var base int = ip
|
||||
/* > 0 */
|
||||
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
|
||||
var distance int = int(base - candidate)
|
||||
/* We have a 5-byte match at ip, and we need to emit bytes in
|
||||
[next_emit, ip). */
|
||||
|
||||
var insert uint = uint(base - next_emit)
|
||||
ip += int(matched)
|
||||
if insert < 6210 {
|
||||
emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
|
||||
emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage)
|
||||
input_size -= uint(base - input)
|
||||
input = base
|
||||
next_emit = input
|
||||
goto next_block
|
||||
} else {
|
||||
emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
}
|
||||
|
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||
if distance == last_distance {
|
||||
writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage)
|
||||
cmd_histo[64]++
|
||||
} else {
|
||||
emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
last_distance = distance
|
||||
}
|
||||
|
||||
emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
|
||||
next_emit = ip
|
||||
if ip >= ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some positions
|
||||
within the last copy. */
|
||||
{
|
||||
var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
|
||||
var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
|
||||
var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
|
||||
candidate = base_ip + table[cur_hash]
|
||||
table[cur_hash] = int(ip - base_ip)
|
||||
}
|
||||
}
|
||||
|
||||
for isMatch5(in[ip:], in[candidate:]) {
|
||||
var base int = ip
|
||||
/* We have a 5-byte match at ip, and no need to emit any literal bytes
|
||||
prior to ip. */
|
||||
|
||||
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
|
||||
if ip-candidate > maxDistance_compress_fragment {
|
||||
break
|
||||
}
|
||||
ip += int(matched)
|
||||
last_distance = int(base - candidate) /* > 0 */
|
||||
emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
|
||||
next_emit = ip
|
||||
if ip >= ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some positions
|
||||
within the last copy. */
|
||||
{
|
||||
var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
|
||||
var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
|
||||
var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
|
||||
candidate = base_ip + table[cur_hash]
|
||||
table[cur_hash] = int(ip - base_ip)
|
||||
}
|
||||
}
|
||||
|
||||
ip++
|
||||
next_hash = hash5(in[ip:], shift)
|
||||
}
|
||||
}
|
||||
|
||||
emit_remainder:
|
||||
assert(next_emit <= ip_end)
|
||||
input += int(block_size)
|
||||
input_size -= block_size
|
||||
block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kMergeBlockSize)
|
||||
|
||||
/* Decide if we want to continue this meta-block instead of emitting the
|
||||
last insert-only command. */
|
||||
if input_size > 0 && total_block_size+block_size <= 1<<20 && shouldMergeBlock(in[input:], block_size, lit_depth[:]) {
|
||||
assert(total_block_size > 1<<16)
|
||||
|
||||
/* Update the size of the current meta-block and continue emitting commands.
|
||||
We can do this because the current size and the new size both have 5
|
||||
nibbles. */
|
||||
total_block_size += block_size
|
||||
|
||||
updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage)
|
||||
goto emit_commands
|
||||
}
|
||||
|
||||
/* Emit the remaining bytes as literals. */
|
||||
if next_emit < ip_end {
|
||||
var insert uint = uint(ip_end - next_emit)
|
||||
if insert < 6210 {
|
||||
emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||
} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
|
||||
emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage)
|
||||
} else {
|
||||
emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
|
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||
}
|
||||
}
|
||||
|
||||
next_emit = ip_end
|
||||
|
||||
/* If we have more data, write a new meta-block header and prefix codes and
|
||||
then continue emitting commands. */
|
||||
next_block:
|
||||
if input_size > 0 {
|
||||
metablock_start = input
|
||||
block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
|
||||
total_block_size = block_size
|
||||
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that
|
||||
we can update it later if we decide to extend this meta-block. */
|
||||
mlen_storage_ix = *storage_ix + 3
|
||||
|
||||
storeMetaBlockHeader1(block_size, false, storage_ix, storage)
|
||||
|
||||
/* No block splits, no contexts. */
|
||||
writeBits(13, 0, storage_ix, storage)
|
||||
|
||||
literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||
buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage)
|
||||
goto emit_commands
|
||||
}
|
||||
|
||||
if !is_last {
|
||||
/* If this is not the last block, update the command and distance prefix
|
||||
codes for the next block and store the compressed forms. */
|
||||
cmd_code[0] = 0
|
||||
|
||||
*cmd_code_numbits = 0
|
||||
buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code)
|
||||
}
|
||||
}
|
||||
|
||||
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
||||
meta-blocks, and updates the "*storage_ix" bit position.
|
||||
|
||||
If "is_last" is 1, emits an additional empty last meta-block.
|
||||
|
||||
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes
|
||||
(see comment in encode.h) used for the encoding of this input fragment.
|
||||
If "is_last" is 0, they are updated to reflect the statistics
|
||||
of this input fragment, to be used for the encoding of the next fragment.
|
||||
|
||||
"*cmd_code_numbits" is the number of bits of the compressed representation
|
||||
of the command and distance prefix codes, and "cmd_code" is an array of
|
||||
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
|
||||
command and distance prefix codes. If "is_last" is 0, these are also
|
||||
updated to represent the updated "cmd_depth" and "cmd_bits".
|
||||
|
||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
||||
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
|
||||
OUTPUT: maximal copy distance <= |input_size|
|
||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
||||
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
|
||||
var initial_storage_ix uint = *storage_ix
|
||||
var table_bits uint = uint(log2FloorNonZero(table_size))
|
||||
|
||||
if input_size == 0 {
|
||||
assert(is_last)
|
||||
writeBits(1, 1, storage_ix, storage) /* islast */
|
||||
writeBits(1, 1, storage_ix, storage) /* isempty */
|
||||
*storage_ix = (*storage_ix + 7) &^ 7
|
||||
return
|
||||
}
|
||||
|
||||
compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage)
|
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */
|
||||
if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
|
||||
emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage)
|
||||
}
|
||||
|
||||
if is_last {
|
||||
writeBits(1, 1, storage_ix, storage) /* islast */
|
||||
writeBits(1, 1, storage_ix, storage) /* isempty */
|
||||
*storage_ix = (*storage_ix + 7) &^ 7
|
||||
}
|
||||
}
|
||||
748
vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go
generated
vendored
Normal file
748
vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go
generated
vendored
Normal file
@ -0,0 +1,748 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input
|
||||
history. This function uses two-pass processing: in the first pass we save
|
||||
the found backward matches and literal bytes into a buffer, and in the
|
||||
second pass we emit them into the bit stream using prefix codes built based
|
||||
on the actual command and literal byte histograms. */
|
||||
|
||||
const kCompressFragmentTwoPassBlockSize uint = 1 << 17
|
||||
|
||||
func hash1(p []byte, shift uint, length uint) uint32 {
|
||||
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
|
||||
return uint32(h >> shift)
|
||||
}
|
||||
|
||||
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
|
||||
assert(offset <= 8-length)
|
||||
{
|
||||
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
|
||||
return uint32(h >> shift)
|
||||
}
|
||||
}
|
||||
|
||||
func isMatch1(p1 []byte, p2 []byte, length uint) bool {
|
||||
if binary.LittleEndian.Uint32(p1) != binary.LittleEndian.Uint32(p2) {
|
||||
return false
|
||||
}
|
||||
if length == 4 {
|
||||
return true
|
||||
}
|
||||
return p1[4] == p2[4] && p1[5] == p2[5]
|
||||
}
|
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
|
||||
"bits" based on "histogram" and stores it into the bit stream. */
|
||||
func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
|
||||
var tree [129]huffmanTree
|
||||
var cmd_depth = [numCommandSymbols]byte{0}
|
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
||||
|
||||
var cmd_bits [64]uint16
|
||||
createHuffmanTree(histogram, 64, 15, tree[:], depth)
|
||||
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
|
||||
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
functions. */
|
||||
copy(cmd_depth[:], depth[24:][:24])
|
||||
|
||||
copy(cmd_depth[24:][:], depth[:8])
|
||||
copy(cmd_depth[32:][:], depth[48:][:8])
|
||||
copy(cmd_depth[40:][:], depth[8:][:8])
|
||||
copy(cmd_depth[48:][:], depth[56:][:8])
|
||||
copy(cmd_depth[56:][:], depth[16:][:8])
|
||||
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
|
||||
copy(bits, cmd_bits[24:][:8])
|
||||
copy(bits[8:], cmd_bits[40:][:8])
|
||||
copy(bits[16:], cmd_bits[56:][:8])
|
||||
copy(bits[24:], cmd_bits[:24])
|
||||
copy(bits[48:], cmd_bits[32:][:8])
|
||||
copy(bits[56:], cmd_bits[48:][:8])
|
||||
convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
|
||||
{
|
||||
/* Create the bit length array for the full command alphabet. */
|
||||
var i uint
|
||||
for i := 0; i < int(64); i++ {
|
||||
cmd_depth[i] = 0
|
||||
} /* only 64 first values were used */
|
||||
copy(cmd_depth[:], depth[24:][:8])
|
||||
copy(cmd_depth[64:][:], depth[32:][:8])
|
||||
copy(cmd_depth[128:][:], depth[40:][:8])
|
||||
copy(cmd_depth[192:][:], depth[48:][:8])
|
||||
copy(cmd_depth[384:][:], depth[56:][:8])
|
||||
for i = 0; i < 8; i++ {
|
||||
cmd_depth[128+8*i] = depth[i]
|
||||
cmd_depth[256+8*i] = depth[8+i]
|
||||
cmd_depth[448+8*i] = depth[16+i]
|
||||
}
|
||||
|
||||
storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
|
||||
}
|
||||
|
||||
storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
|
||||
}
|
||||
|
||||
func emitInsertLen(insertlen uint32, commands *[]uint32) {
|
||||
if insertlen < 6 {
|
||||
(*commands)[0] = insertlen
|
||||
} else if insertlen < 130 {
|
||||
var tail uint32 = insertlen - 2
|
||||
var nbits uint32 = log2FloorNonZero(uint(tail)) - 1
|
||||
var prefix uint32 = tail >> nbits
|
||||
var inscode uint32 = (nbits << 1) + prefix + 2
|
||||
var extra uint32 = tail - (prefix << nbits)
|
||||
(*commands)[0] = inscode | extra<<8
|
||||
} else if insertlen < 2114 {
|
||||
var tail uint32 = insertlen - 66
|
||||
var nbits uint32 = log2FloorNonZero(uint(tail))
|
||||
var code uint32 = nbits + 10
|
||||
var extra uint32 = tail - (1 << nbits)
|
||||
(*commands)[0] = code | extra<<8
|
||||
} else if insertlen < 6210 {
|
||||
var extra uint32 = insertlen - 2114
|
||||
(*commands)[0] = 21 | extra<<8
|
||||
} else if insertlen < 22594 {
|
||||
var extra uint32 = insertlen - 6210
|
||||
(*commands)[0] = 22 | extra<<8
|
||||
} else {
|
||||
var extra uint32 = insertlen - 22594
|
||||
(*commands)[0] = 23 | extra<<8
|
||||
}
|
||||
|
||||
*commands = (*commands)[1:]
|
||||
}
|
||||
|
||||
func emitCopyLen(copylen uint, commands *[]uint32) {
|
||||
if copylen < 10 {
|
||||
(*commands)[0] = uint32(copylen + 38)
|
||||
} else if copylen < 134 {
|
||||
var tail uint = copylen - 6
|
||||
var nbits uint = uint(log2FloorNonZero(tail) - 1)
|
||||
var prefix uint = tail >> nbits
|
||||
var code uint = (nbits << 1) + prefix + 44
|
||||
var extra uint = tail - (prefix << nbits)
|
||||
(*commands)[0] = uint32(code | extra<<8)
|
||||
} else if copylen < 2118 {
|
||||
var tail uint = copylen - 70
|
||||
var nbits uint = uint(log2FloorNonZero(tail))
|
||||
var code uint = nbits + 52
|
||||
var extra uint = tail - (uint(1) << nbits)
|
||||
(*commands)[0] = uint32(code | extra<<8)
|
||||
} else {
|
||||
var extra uint = copylen - 2118
|
||||
(*commands)[0] = uint32(63 | extra<<8)
|
||||
}
|
||||
|
||||
*commands = (*commands)[1:]
|
||||
}
|
||||
|
||||
func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
|
||||
if copylen < 12 {
|
||||
(*commands)[0] = uint32(copylen + 20)
|
||||
*commands = (*commands)[1:]
|
||||
} else if copylen < 72 {
|
||||
var tail uint = copylen - 8
|
||||
var nbits uint = uint(log2FloorNonZero(tail) - 1)
|
||||
var prefix uint = tail >> nbits
|
||||
var code uint = (nbits << 1) + prefix + 28
|
||||
var extra uint = tail - (prefix << nbits)
|
||||
(*commands)[0] = uint32(code | extra<<8)
|
||||
*commands = (*commands)[1:]
|
||||
} else if copylen < 136 {
|
||||
var tail uint = copylen - 8
|
||||
var code uint = (tail >> 5) + 54
|
||||
var extra uint = tail & 31
|
||||
(*commands)[0] = uint32(code | extra<<8)
|
||||
*commands = (*commands)[1:]
|
||||
(*commands)[0] = 64
|
||||
*commands = (*commands)[1:]
|
||||
} else if copylen < 2120 {
|
||||
var tail uint = copylen - 72
|
||||
var nbits uint = uint(log2FloorNonZero(tail))
|
||||
var code uint = nbits + 52
|
||||
var extra uint = tail - (uint(1) << nbits)
|
||||
(*commands)[0] = uint32(code | extra<<8)
|
||||
*commands = (*commands)[1:]
|
||||
(*commands)[0] = 64
|
||||
*commands = (*commands)[1:]
|
||||
} else {
|
||||
var extra uint = copylen - 2120
|
||||
(*commands)[0] = uint32(63 | extra<<8)
|
||||
*commands = (*commands)[1:]
|
||||
(*commands)[0] = 64
|
||||
*commands = (*commands)[1:]
|
||||
}
|
||||
}
|
||||
|
||||
func emitDistance(distance uint32, commands *[]uint32) {
|
||||
var d uint32 = distance + 3
|
||||
var nbits uint32 = log2FloorNonZero(uint(d)) - 1
|
||||
var prefix uint32 = (d >> nbits) & 1
|
||||
var offset uint32 = (2 + prefix) << nbits
|
||||
var distcode uint32 = 2*(nbits-1) + prefix + 80
|
||||
var extra uint32 = d - offset
|
||||
(*commands)[0] = distcode | extra<<8
|
||||
*commands = (*commands)[1:]
|
||||
}
|
||||
|
||||
/* REQUIRES: len <= 1 << 24. */
|
||||
func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
|
||||
var nibbles uint = 6
|
||||
|
||||
/* ISLAST */
|
||||
writeBits(1, 0, storage_ix, storage)
|
||||
|
||||
if len <= 1<<16 {
|
||||
nibbles = 4
|
||||
} else if len <= 1<<20 {
|
||||
nibbles = 5
|
||||
}
|
||||
|
||||
writeBits(2, uint64(nibbles)-4, storage_ix, storage)
|
||||
writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
|
||||
|
||||
/* ISUNCOMPRESSED */
|
||||
writeSingleBit(is_uncompressed, storage_ix, storage)
|
||||
}
|
||||
|
||||
func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) {
|
||||
var ip int = 0
|
||||
var shift uint = 64 - table_bits
|
||||
var ip_end int = int(block_size)
|
||||
var base_ip int = -cap(base_ip_ptr) + cap(input)
|
||||
var next_emit int = 0
|
||||
var last_distance int = -1
|
||||
/* "ip" is the input pointer. */
|
||||
|
||||
const kInputMarginBytes uint = windowGap
|
||||
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a
|
||||
previous copy. Bytes between "next_emit" and the start of the next copy or
|
||||
the end of the input will be emitted as literal bytes. */
|
||||
if block_size >= kInputMarginBytes {
|
||||
var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes)
|
||||
var ip_limit int = int(len_limit)
|
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be
|
||||
sure that all distances are at most window size - 16.
|
||||
For all other blocks, we only need to keep a margin of 5 bytes so that
|
||||
we don't go over the block size with a copy. */
|
||||
|
||||
var next_hash uint32
|
||||
ip++
|
||||
for next_hash = hash1(input[ip:], shift, min_match); ; {
|
||||
var skip uint32 = 32
|
||||
var next_ip int = ip
|
||||
/* Step 1: Scan forward in the input looking for a 6-byte-long match.
|
||||
If we get close to exhausting the input then goto emit_remainder.
|
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches
|
||||
found, start looking only at every other byte. If 32 more bytes are
|
||||
scanned, look at every third byte, etc.. When a match is found,
|
||||
immediately go back to looking at every byte. This is a small loss
|
||||
(~5% performance, ~0.1% density) for compressible data due to more
|
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge
|
||||
win since the compressor quickly "realizes" the data is incompressible
|
||||
and doesn't bother looking for matches everywhere.
|
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the
|
||||
last match; dividing it by 32 (ie. right-shifting by five) gives the
|
||||
number of bytes to move ahead for each iteration. */
|
||||
|
||||
var candidate int
|
||||
|
||||
assert(next_emit < ip)
|
||||
|
||||
trawl:
|
||||
for {
|
||||
var hash uint32 = next_hash
|
||||
var bytes_between_hash_lookups uint32 = skip >> 5
|
||||
skip++
|
||||
ip = next_ip
|
||||
assert(hash == hash1(input[ip:], shift, min_match))
|
||||
next_ip = int(uint32(ip) + bytes_between_hash_lookups)
|
||||
if next_ip > ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
|
||||
next_hash = hash1(input[next_ip:], shift, min_match)
|
||||
candidate = ip - last_distance
|
||||
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
||||
if candidate < ip {
|
||||
table[hash] = int(ip - base_ip)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
candidate = base_ip + table[hash]
|
||||
assert(candidate >= base_ip)
|
||||
assert(candidate < ip)
|
||||
|
||||
table[hash] = int(ip - base_ip)
|
||||
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search.
|
||||
Checking is done outside of hot loop to reduce overhead. */
|
||||
if ip-candidate > maxDistance_compress_fragment {
|
||||
goto trawl
|
||||
}
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit", and then see if we can find a next match immediately
|
||||
afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
{
|
||||
var base int = ip
|
||||
/* > 0 */
|
||||
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
|
||||
var distance int = int(base - candidate)
|
||||
/* We have a 6-byte match at ip, and we need to emit bytes in
|
||||
[next_emit, ip). */
|
||||
|
||||
var insert int = int(base - next_emit)
|
||||
ip += int(matched)
|
||||
emitInsertLen(uint32(insert), commands)
|
||||
copy(*literals, input[next_emit:][:uint(insert)])
|
||||
*literals = (*literals)[insert:]
|
||||
if distance == last_distance {
|
||||
(*commands)[0] = 64
|
||||
*commands = (*commands)[1:]
|
||||
} else {
|
||||
emitDistance(uint32(distance), commands)
|
||||
last_distance = distance
|
||||
}
|
||||
|
||||
emitCopyLenLastDistance(matched, commands)
|
||||
|
||||
next_emit = ip
|
||||
if ip >= ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
{
|
||||
var input_bytes uint64
|
||||
var cur_hash uint32
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some
|
||||
positions within the last copy. */
|
||||
|
||||
var prev_hash uint32
|
||||
if min_match == 4 {
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
|
||||
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
} else {
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 5)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 4)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
|
||||
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
}
|
||||
|
||||
candidate = base_ip + table[cur_hash]
|
||||
table[cur_hash] = int(ip - base_ip)
|
||||
}
|
||||
}
|
||||
|
||||
for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
|
||||
var base int = ip
|
||||
/* We have a 6-byte match at ip, and no need to emit any
|
||||
literal bytes prior to ip. */
|
||||
|
||||
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
|
||||
ip += int(matched)
|
||||
last_distance = int(base - candidate) /* > 0 */
|
||||
emitCopyLen(matched, commands)
|
||||
emitDistance(uint32(last_distance), commands)
|
||||
|
||||
next_emit = ip
|
||||
if ip >= ip_limit {
|
||||
goto emit_remainder
|
||||
}
|
||||
{
|
||||
var input_bytes uint64
|
||||
var cur_hash uint32
|
||||
/* We could immediately start working at ip now, but to improve
|
||||
compression we first update "table" with the hashes of some
|
||||
positions within the last copy. */
|
||||
|
||||
var prev_hash uint32
|
||||
if min_match == 4 {
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
|
||||
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
} else {
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 5)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 4)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 3)
|
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
|
||||
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 2)
|
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
|
||||
table[prev_hash] = int(ip - base_ip - 1)
|
||||
}
|
||||
|
||||
candidate = base_ip + table[cur_hash]
|
||||
table[cur_hash] = int(ip - base_ip)
|
||||
}
|
||||
}
|
||||
|
||||
ip++
|
||||
next_hash = hash1(input[ip:], shift, min_match)
|
||||
}
|
||||
}
|
||||
|
||||
emit_remainder:
|
||||
assert(next_emit <= ip_end)
|
||||
|
||||
/* Emit the remaining bytes as literals. */
|
||||
if next_emit < ip_end {
|
||||
var insert uint32 = uint32(ip_end - next_emit)
|
||||
emitInsertLen(insert, commands)
|
||||
copy(*literals, input[next_emit:][:insert])
|
||||
*literals = (*literals)[insert:]
|
||||
}
|
||||
}
|
||||
|
||||
var storeCommands_kNumExtraBits = [128]uint32{
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
5,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
12,
|
||||
14,
|
||||
24,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
5,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
24,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
4,
|
||||
4,
|
||||
5,
|
||||
5,
|
||||
6,
|
||||
6,
|
||||
7,
|
||||
7,
|
||||
8,
|
||||
8,
|
||||
9,
|
||||
9,
|
||||
10,
|
||||
10,
|
||||
11,
|
||||
11,
|
||||
12,
|
||||
12,
|
||||
13,
|
||||
13,
|
||||
14,
|
||||
14,
|
||||
15,
|
||||
15,
|
||||
16,
|
||||
16,
|
||||
17,
|
||||
17,
|
||||
18,
|
||||
18,
|
||||
19,
|
||||
19,
|
||||
20,
|
||||
20,
|
||||
21,
|
||||
21,
|
||||
22,
|
||||
22,
|
||||
23,
|
||||
23,
|
||||
24,
|
||||
24,
|
||||
}
|
||||
var storeCommands_kInsertOffset = [24]uint32{
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
8,
|
||||
10,
|
||||
14,
|
||||
18,
|
||||
26,
|
||||
34,
|
||||
50,
|
||||
66,
|
||||
98,
|
||||
130,
|
||||
194,
|
||||
322,
|
||||
578,
|
||||
1090,
|
||||
2114,
|
||||
6210,
|
||||
22594,
|
||||
}
|
||||
|
||||
func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) {
|
||||
var lit_depths [256]byte
|
||||
var lit_bits [256]uint16
|
||||
var lit_histo = [256]uint32{0}
|
||||
var cmd_depths = [128]byte{0}
|
||||
var cmd_bits = [128]uint16{0}
|
||||
var cmd_histo = [128]uint32{0}
|
||||
var i uint
|
||||
for i = 0; i < num_literals; i++ {
|
||||
lit_histo[literals[i]]++
|
||||
}
|
||||
|
||||
buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */
|
||||
8, lit_depths[:], lit_bits[:], storage_ix, storage)
|
||||
|
||||
for i = 0; i < num_commands; i++ {
|
||||
var code uint32 = commands[i] & 0xFF
|
||||
assert(code < 128)
|
||||
cmd_histo[code]++
|
||||
}
|
||||
|
||||
cmd_histo[1] += 1
|
||||
cmd_histo[2] += 1
|
||||
cmd_histo[64] += 1
|
||||
cmd_histo[84] += 1
|
||||
buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage)
|
||||
|
||||
for i = 0; i < num_commands; i++ {
|
||||
var cmd uint32 = commands[i]
|
||||
var code uint32 = cmd & 0xFF
|
||||
var extra uint32 = cmd >> 8
|
||||
assert(code < 128)
|
||||
writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage)
|
||||
writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage)
|
||||
if code < 24 {
|
||||
var insert uint32 = storeCommands_kInsertOffset[code] + extra
|
||||
var j uint32
|
||||
for j = 0; j < insert; j++ {
|
||||
var lit byte = literals[0]
|
||||
writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage)
|
||||
literals = literals[1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Acceptable loss for uncompressible speedup is 2% */
|
||||
const minRatio = 0.98
|
||||
|
||||
const sampleRate = 43
|
||||
|
||||
func shouldCompress(input []byte, input_size uint, num_literals uint) bool {
|
||||
var corpus_size float64 = float64(input_size)
|
||||
if float64(num_literals) < minRatio*corpus_size {
|
||||
return true
|
||||
} else {
|
||||
var literal_histo = [256]uint32{0}
|
||||
var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate
|
||||
var i uint
|
||||
for i = 0; i < input_size; i += sampleRate {
|
||||
literal_histo[input[i]]++
|
||||
}
|
||||
|
||||
return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost
|
||||
}
|
||||
}
|
||||
|
||||
func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) {
|
||||
var bitpos uint = new_storage_ix & 7
|
||||
var mask uint = (1 << bitpos) - 1
|
||||
storage[new_storage_ix>>3] &= byte(mask)
|
||||
*storage_ix = new_storage_ix
|
||||
}
|
||||
|
||||
func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) {
|
||||
storeMetaBlockHeader(input_size, true, storage_ix, storage)
|
||||
*storage_ix = (*storage_ix + 7) &^ 7
|
||||
copy(storage[*storage_ix>>3:], input[:input_size])
|
||||
*storage_ix += input_size << 3
|
||||
storage[*storage_ix>>3] = 0
|
||||
}
|
||||
|
||||
func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) {
|
||||
/* Save the start of the first block for position and distance computations.
|
||||
*/
|
||||
var base_ip []byte = input
|
||||
|
||||
for input_size > 0 {
|
||||
var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize)
|
||||
var commands []uint32 = command_buf
|
||||
var literals []byte = literal_buf
|
||||
var num_literals uint
|
||||
createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands)
|
||||
num_literals = uint(-cap(literals) + cap(literal_buf))
|
||||
if shouldCompress(input, block_size, num_literals) {
|
||||
var num_commands uint = uint(-cap(commands) + cap(command_buf))
|
||||
storeMetaBlockHeader(block_size, false, storage_ix, storage)
|
||||
|
||||
/* No block splits, no contexts. */
|
||||
writeBits(13, 0, storage_ix, storage)
|
||||
|
||||
storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage)
|
||||
} else {
|
||||
/* Since we did not find many backward references and the entropy of
|
||||
the data is close to 8 bits, we can simply emit an uncompressed block.
|
||||
This makes compression speed of uncompressible data about 3x faster. */
|
||||
emitUncompressedMetaBlock(input, block_size, storage_ix, storage)
|
||||
}
|
||||
|
||||
input = input[block_size:]
|
||||
input_size -= block_size
|
||||
}
|
||||
}
|
||||
|
||||
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
||||
meta-blocks, and updates the "*storage_ix" bit position.
|
||||
|
||||
If "is_last" is 1, emits an additional empty last meta-block.
|
||||
|
||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
||||
REQUIRES: "command_buf" and "literal_buf" point to at least
|
||||
kCompressFragmentTwoPassBlockSize long arrays.
|
||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
||||
REQUIRES: "table_size" is a power of two
|
||||
OUTPUT: maximal copy distance <= |input_size|
|
||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
||||
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
|
||||
var initial_storage_ix uint = *storage_ix
|
||||
var table_bits uint = uint(log2FloorNonZero(table_size))
|
||||
var min_match uint
|
||||
if table_bits <= 15 {
|
||||
min_match = 4
|
||||
} else {
|
||||
min_match = 6
|
||||
}
|
||||
compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage)
|
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */
|
||||
if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
|
||||
rewindBitPosition(initial_storage_ix, storage_ix, storage)
|
||||
emitUncompressedMetaBlock(input, input_size, storage_ix, storage)
|
||||
}
|
||||
|
||||
if is_last {
|
||||
writeBits(1, 1, storage_ix, storage) /* islast */
|
||||
writeBits(1, 1, storage_ix, storage) /* isempty */
|
||||
*storage_ix = (*storage_ix + 7) &^ 7
|
||||
}
|
||||
}
|
||||
77
vendor/github.com/andybalholm/brotli/constants.go
generated
vendored
Normal file
77
vendor/github.com/andybalholm/brotli/constants.go
generated
vendored
Normal file
@ -0,0 +1,77 @@
|
||||
package brotli
|
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Specification: 7.3. Encoding of the context map */
|
||||
const contextMapMaxRle = 16
|
||||
|
||||
/* Specification: 2. Compressed representation overview */
|
||||
const maxNumberOfBlockTypes = 256
|
||||
|
||||
/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
|
||||
const numLiteralSymbols = 256
|
||||
|
||||
const numCommandSymbols = 704
|
||||
|
||||
const numBlockLenSymbols = 26
|
||||
|
||||
const maxContextMapSymbols = (maxNumberOfBlockTypes + contextMapMaxRle)
|
||||
|
||||
const maxBlockTypeSymbols = (maxNumberOfBlockTypes + 2)
|
||||
|
||||
/* Specification: 3.5. Complex prefix codes */
|
||||
const repeatPreviousCodeLength = 16
|
||||
|
||||
const repeatZeroCodeLength = 17
|
||||
|
||||
const codeLengthCodes = (repeatZeroCodeLength + 1)
|
||||
|
||||
/* "code length of 8 is repeated" */
|
||||
const initialRepeatedCodeLength = 8
|
||||
|
||||
/* "Large Window Brotli" */
|
||||
const largeMaxDistanceBits = 62
|
||||
|
||||
const largeMinWbits = 10
|
||||
|
||||
const largeMaxWbits = 30
|
||||
|
||||
/* Specification: 4. Encoding of distances */
|
||||
const numDistanceShortCodes = 16
|
||||
|
||||
const maxNpostfix = 3
|
||||
|
||||
const maxNdirect = 120
|
||||
|
||||
const maxDistanceBits = 24
|
||||
|
||||
func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint {
|
||||
return numDistanceShortCodes + NDIRECT + uint(MAXNBITS<<(NPOSTFIX+1))
|
||||
}
|
||||
|
||||
/* numDistanceSymbols == 1128 */
|
||||
const numDistanceSymbols = 1128
|
||||
|
||||
const maxDistance = 0x3FFFFFC
|
||||
|
||||
const maxAllowedDistance = 0x7FFFFFFC
|
||||
|
||||
/* 7.1. Context modes and context ID lookup for literals */
|
||||
/* "context IDs for literals are in the range of 0..63" */
|
||||
const literalContextBits = 6
|
||||
|
||||
/* 7.2. Context ID for distances */
|
||||
const distanceContextBits = 2
|
||||
|
||||
/* 9.1. Format of the Stream Header */
|
||||
/* Number of slack bytes for window size. Don't confuse
|
||||
with BROTLI_NUM_DISTANCE_SHORT_CODES. */
|
||||
const windowGap = 16
|
||||
|
||||
func maxBackwardLimit(W uint) uint {
|
||||
return (uint(1) << W) - windowGap
|
||||
}
|
||||
2176
vendor/github.com/andybalholm/brotli/context.go
generated
vendored
Normal file
2176
vendor/github.com/andybalholm/brotli/context.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
2586
vendor/github.com/andybalholm/brotli/decode.go
generated
vendored
Normal file
2586
vendor/github.com/andybalholm/brotli/decode.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
122890
vendor/github.com/andybalholm/brotli/dictionary.go
generated
vendored
Normal file
122890
vendor/github.com/andybalholm/brotli/dictionary.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
32779
vendor/github.com/andybalholm/brotli/dictionary_hash.go
generated
vendored
Normal file
32779
vendor/github.com/andybalholm/brotli/dictionary_hash.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1220
vendor/github.com/andybalholm/brotli/encode.go
generated
vendored
Normal file
1220
vendor/github.com/andybalholm/brotli/encode.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
22
vendor/github.com/andybalholm/brotli/encoder_dict.go
generated
vendored
Normal file
22
vendor/github.com/andybalholm/brotli/encoder_dict.go
generated
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
package brotli
|
||||
|
||||
/* Dictionary data (words and transforms) for 1 possible context */
|
||||
type encoderDictionary struct {
|
||||
words *dictionary
|
||||
cutoffTransformsCount uint32
|
||||
cutoffTransforms uint64
|
||||
hash_table []uint16
|
||||
buckets []uint16
|
||||
dict_words []dictWord
|
||||
}
|
||||
|
||||
func initEncoderDictionary(dict *encoderDictionary) {
|
||||
dict.words = getDictionary()
|
||||
|
||||
dict.hash_table = kStaticDictionaryHash[:]
|
||||
dict.buckets = kStaticDictionaryBuckets[:]
|
||||
dict.dict_words = kStaticDictionaryWords[:]
|
||||
|
||||
dict.cutoffTransformsCount = kCutoffTransformsCount
|
||||
dict.cutoffTransforms = kCutoffTransforms
|
||||
}
|
||||
592
vendor/github.com/andybalholm/brotli/entropy_encode.go
generated
vendored
Normal file
592
vendor/github.com/andybalholm/brotli/entropy_encode.go
generated
vendored
Normal file
@ -0,0 +1,592 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Entropy encoding (Huffman) utilities. */
|
||||
|
||||
/* A node of a Huffman tree. */
|
||||
type huffmanTree struct {
|
||||
total_count_ uint32
|
||||
index_left_ int16
|
||||
index_right_or_value_ int16
|
||||
}
|
||||
|
||||
func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
|
||||
self.total_count_ = count
|
||||
self.index_left_ = left
|
||||
self.index_right_or_value_ = right
|
||||
}
|
||||
|
||||
/* Input size optimized Shell sort. */
|
||||
type huffmanTreeComparator func(huffmanTree, huffmanTree) bool
|
||||
|
||||
var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1}
|
||||
|
||||
func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) {
|
||||
if n < 13 {
|
||||
/* Insertion sort. */
|
||||
var i uint
|
||||
for i = 1; i < n; i++ {
|
||||
var tmp huffmanTree = items[i]
|
||||
var k uint = i
|
||||
var j uint = i - 1
|
||||
for comparator(tmp, items[j]) {
|
||||
items[k] = items[j]
|
||||
k = j
|
||||
if j == 0 {
|
||||
break
|
||||
}
|
||||
j--
|
||||
}
|
||||
|
||||
items[k] = tmp
|
||||
}
|
||||
|
||||
return
|
||||
} else {
|
||||
var g int
|
||||
if n < 57 {
|
||||
g = 2
|
||||
} else {
|
||||
g = 0
|
||||
}
|
||||
for ; g < 6; g++ {
|
||||
var gap uint = sortHuffmanTreeItems_gaps[g]
|
||||
var i uint
|
||||
for i = gap; i < n; i++ {
|
||||
var j uint = i
|
||||
var tmp huffmanTree = items[i]
|
||||
for ; j >= gap && comparator(tmp, items[j-gap]); j -= gap {
|
||||
items[j] = items[j-gap]
|
||||
}
|
||||
|
||||
items[j] = tmp
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Returns 1 if assignment of depths succeeded, otherwise 0. */
|
||||
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
|
||||
var stack [16]int
|
||||
var level int = 0
|
||||
var p int = p0
|
||||
assert(max_depth <= 15)
|
||||
stack[0] = -1
|
||||
for {
|
||||
if pool[p].index_left_ >= 0 {
|
||||
level++
|
||||
if level > max_depth {
|
||||
return false
|
||||
}
|
||||
stack[level] = int(pool[p].index_right_or_value_)
|
||||
p = int(pool[p].index_left_)
|
||||
continue
|
||||
} else {
|
||||
depth[pool[p].index_right_or_value_] = byte(level)
|
||||
}
|
||||
|
||||
for level >= 0 && stack[level] == -1 {
|
||||
level--
|
||||
}
|
||||
if level < 0 {
|
||||
return true
|
||||
}
|
||||
p = stack[level]
|
||||
stack[level] = -1
|
||||
}
|
||||
}
|
||||
|
||||
/* Sort the root nodes, least popular first. */
|
||||
func sortHuffmanTree(v0 huffmanTree, v1 huffmanTree) bool {
|
||||
if v0.total_count_ != v1.total_count_ {
|
||||
return v0.total_count_ < v1.total_count_
|
||||
}
|
||||
|
||||
return v0.index_right_or_value_ > v1.index_right_or_value_
|
||||
}
|
||||
|
||||
/* This function will create a Huffman tree.
|
||||
|
||||
The catch here is that the tree cannot be arbitrarily deep.
|
||||
Brotli specifies a maximum depth of 15 bits for "code trees"
|
||||
and 7 bits for "code length code trees."
|
||||
|
||||
count_limit is the value that is to be faked as the minimum value
|
||||
and this minimum value is raised until the tree matches the
|
||||
maximum length requirement.
|
||||
|
||||
This algorithm is not of excellent performance for very long data blocks,
|
||||
especially when population counts are longer than 2**tree_limit, but
|
||||
we are not planning to use this with extremely long blocks.
|
||||
|
||||
See http://en.wikipedia.org/wiki/Huffman_coding */
|
||||
func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) {
|
||||
var count_limit uint32
|
||||
var sentinel huffmanTree
|
||||
initHuffmanTree(&sentinel, math.MaxUint32, -1, -1)
|
||||
|
||||
/* For block sizes below 64 kB, we never need to do a second iteration
|
||||
of this loop. Probably all of our block sizes will be smaller than
|
||||
that, so this loop is mostly of academic interest. If we actually
|
||||
would need this, we would be better off with the Katajainen algorithm. */
|
||||
for count_limit = 1; ; count_limit *= 2 {
|
||||
var n uint = 0
|
||||
var i uint
|
||||
var j uint
|
||||
var k uint
|
||||
for i = length; i != 0; {
|
||||
i--
|
||||
if data[i] != 0 {
|
||||
var count uint32 = brotli_max_uint32_t(data[i], count_limit)
|
||||
initHuffmanTree(&tree[n], count, -1, int16(i))
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
if n == 1 {
|
||||
depth[tree[0].index_right_or_value_] = 1 /* Only one element. */
|
||||
break
|
||||
}
|
||||
|
||||
sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree))
|
||||
|
||||
/* The nodes are:
|
||||
[0, n): the sorted leaf nodes that we start with.
|
||||
[n]: we add a sentinel here.
|
||||
[n + 1, 2n): new parent nodes are added here, starting from
|
||||
(n+1). These are naturally in ascending order.
|
||||
[2n]: we add a sentinel at the end as well.
|
||||
There will be (2n+1) elements at the end. */
|
||||
tree[n] = sentinel
|
||||
|
||||
tree[n+1] = sentinel
|
||||
|
||||
i = 0 /* Points to the next leaf node. */
|
||||
j = n + 1 /* Points to the next non-leaf node. */
|
||||
for k = n - 1; k != 0; k-- {
|
||||
var left uint
|
||||
var right uint
|
||||
if tree[i].total_count_ <= tree[j].total_count_ {
|
||||
left = i
|
||||
i++
|
||||
} else {
|
||||
left = j
|
||||
j++
|
||||
}
|
||||
|
||||
if tree[i].total_count_ <= tree[j].total_count_ {
|
||||
right = i
|
||||
i++
|
||||
} else {
|
||||
right = j
|
||||
j++
|
||||
}
|
||||
{
|
||||
/* The sentinel node becomes the parent node. */
|
||||
var j_end uint = 2*n - k
|
||||
tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_
|
||||
tree[j_end].index_left_ = int16(left)
|
||||
tree[j_end].index_right_or_value_ = int16(right)
|
||||
|
||||
/* Add back the last sentinel node. */
|
||||
tree[j_end+1] = sentinel
|
||||
}
|
||||
}
|
||||
|
||||
if setDepth(int(2*n-1), tree[0:], depth, tree_limit) {
|
||||
/* We need to pack the Huffman tree in tree_limit bits. If this was not
|
||||
successful, add fake entities to the lowest values and retry. */
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reverse(v []byte, start uint, end uint) {
|
||||
end--
|
||||
for start < end {
|
||||
var tmp byte = v[start]
|
||||
v[start] = v[end]
|
||||
v[end] = tmp
|
||||
start++
|
||||
end--
|
||||
}
|
||||
}
|
||||
|
||||
func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
|
||||
assert(repetitions > 0)
|
||||
if previous_value != value {
|
||||
tree[*tree_size] = value
|
||||
extra_bits_data[*tree_size] = 0
|
||||
(*tree_size)++
|
||||
repetitions--
|
||||
}
|
||||
|
||||
if repetitions == 7 {
|
||||
tree[*tree_size] = value
|
||||
extra_bits_data[*tree_size] = 0
|
||||
(*tree_size)++
|
||||
repetitions--
|
||||
}
|
||||
|
||||
if repetitions < 3 {
|
||||
var i uint
|
||||
for i = 0; i < repetitions; i++ {
|
||||
tree[*tree_size] = value
|
||||
extra_bits_data[*tree_size] = 0
|
||||
(*tree_size)++
|
||||
}
|
||||
} else {
|
||||
var start uint = *tree_size
|
||||
repetitions -= 3
|
||||
for {
|
||||
tree[*tree_size] = repeatPreviousCodeLength
|
||||
extra_bits_data[*tree_size] = byte(repetitions & 0x3)
|
||||
(*tree_size)++
|
||||
repetitions >>= 2
|
||||
if repetitions == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
repetitions--
|
||||
}
|
||||
|
||||
reverse(tree, start, *tree_size)
|
||||
reverse(extra_bits_data, start, *tree_size)
|
||||
}
|
||||
}
|
||||
|
||||
func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
|
||||
if repetitions == 11 {
|
||||
tree[*tree_size] = 0
|
||||
extra_bits_data[*tree_size] = 0
|
||||
(*tree_size)++
|
||||
repetitions--
|
||||
}
|
||||
|
||||
if repetitions < 3 {
|
||||
var i uint
|
||||
for i = 0; i < repetitions; i++ {
|
||||
tree[*tree_size] = 0
|
||||
extra_bits_data[*tree_size] = 0
|
||||
(*tree_size)++
|
||||
}
|
||||
} else {
|
||||
var start uint = *tree_size
|
||||
repetitions -= 3
|
||||
for {
|
||||
tree[*tree_size] = repeatZeroCodeLength
|
||||
extra_bits_data[*tree_size] = byte(repetitions & 0x7)
|
||||
(*tree_size)++
|
||||
repetitions >>= 3
|
||||
if repetitions == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
repetitions--
|
||||
}
|
||||
|
||||
reverse(tree, start, *tree_size)
|
||||
reverse(extra_bits_data, start, *tree_size)
|
||||
}
|
||||
}
|
||||
|
||||
/* Change the population counts in a way that the consequent
|
||||
Huffman tree compression, especially its RLE-part will be more
|
||||
likely to compress this data more efficiently.
|
||||
|
||||
length contains the size of the histogram.
|
||||
counts contains the population counts.
|
||||
good_for_rle is a buffer of at least length size */
|
||||
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
|
||||
var nonzero_count uint = 0
|
||||
var stride uint
|
||||
var limit uint
|
||||
var sum uint
|
||||
var streak_limit uint = 1240
|
||||
var i uint
|
||||
/* Let's make the Huffman code more compatible with RLE encoding. */
|
||||
for i = 0; i < length; i++ {
|
||||
if counts[i] != 0 {
|
||||
nonzero_count++
|
||||
}
|
||||
}
|
||||
|
||||
if nonzero_count < 16 {
|
||||
return
|
||||
}
|
||||
|
||||
for length != 0 && counts[length-1] == 0 {
|
||||
length--
|
||||
}
|
||||
|
||||
if length == 0 {
|
||||
return /* All zeros. */
|
||||
}
|
||||
|
||||
/* Now counts[0..length - 1] does not have trailing zeros. */
|
||||
{
|
||||
var nonzeros uint = 0
|
||||
var smallest_nonzero uint32 = 1 << 30
|
||||
for i = 0; i < length; i++ {
|
||||
if counts[i] != 0 {
|
||||
nonzeros++
|
||||
if smallest_nonzero > counts[i] {
|
||||
smallest_nonzero = counts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if nonzeros < 5 {
|
||||
/* Small histogram will model it well. */
|
||||
return
|
||||
}
|
||||
|
||||
if smallest_nonzero < 4 {
|
||||
var zeros uint = length - nonzeros
|
||||
if zeros < 6 {
|
||||
for i = 1; i < length-1; i++ {
|
||||
if counts[i-1] != 0 && counts[i] == 0 && counts[i+1] != 0 {
|
||||
counts[i] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if nonzeros < 28 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
/* 2) Let's mark all population counts that already can be encoded
|
||||
with an RLE code. */
|
||||
for i := 0; i < int(length); i++ {
|
||||
good_for_rle[i] = 0
|
||||
}
|
||||
{
|
||||
var symbol uint32 = counts[0]
|
||||
/* Let's not spoil any of the existing good RLE codes.
|
||||
Mark any seq of 0's that is longer as 5 as a good_for_rle.
|
||||
Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
|
||||
|
||||
var step uint = 0
|
||||
for i = 0; i <= length; i++ {
|
||||
if i == length || counts[i] != symbol {
|
||||
if (symbol == 0 && step >= 5) || (symbol != 0 && step >= 7) {
|
||||
var k uint
|
||||
for k = 0; k < step; k++ {
|
||||
good_for_rle[i-k-1] = 1
|
||||
}
|
||||
}
|
||||
|
||||
step = 1
|
||||
if i != length {
|
||||
symbol = counts[i]
|
||||
}
|
||||
} else {
|
||||
step++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* 3) Let's replace those population counts that lead to more RLE codes.
|
||||
Math here is in 24.8 fixed point representation. */
|
||||
stride = 0
|
||||
|
||||
limit = uint(256*(counts[0]+counts[1]+counts[2])/3 + 420)
|
||||
sum = 0
|
||||
for i = 0; i <= length; i++ {
|
||||
if i == length || good_for_rle[i] != 0 || (i != 0 && good_for_rle[i-1] != 0) || (256*counts[i]-uint32(limit)+uint32(streak_limit)) >= uint32(2*streak_limit) {
|
||||
if stride >= 4 || (stride >= 3 && sum == 0) {
|
||||
var k uint
|
||||
var count uint = (sum + stride/2) / stride
|
||||
/* The stride must end, collapse what we have, if we have enough (4). */
|
||||
if count == 0 {
|
||||
count = 1
|
||||
}
|
||||
|
||||
if sum == 0 {
|
||||
/* Don't make an all zeros stride to be upgraded to ones. */
|
||||
count = 0
|
||||
}
|
||||
|
||||
for k = 0; k < stride; k++ {
|
||||
/* We don't want to change value at counts[i],
|
||||
that is already belonging to the next stride. Thus - 1. */
|
||||
counts[i-k-1] = uint32(count)
|
||||
}
|
||||
}
|
||||
|
||||
stride = 0
|
||||
sum = 0
|
||||
if i < length-2 {
|
||||
/* All interesting strides have a count of at least 4, */
|
||||
/* at least when non-zeros. */
|
||||
limit = uint(256*(counts[i]+counts[i+1]+counts[i+2])/3 + 420)
|
||||
} else if i < length {
|
||||
limit = uint(256 * counts[i])
|
||||
} else {
|
||||
limit = 0
|
||||
}
|
||||
}
|
||||
|
||||
stride++
|
||||
if i != length {
|
||||
sum += uint(counts[i])
|
||||
if stride >= 4 {
|
||||
limit = (256*sum + stride/2) / stride
|
||||
}
|
||||
|
||||
if stride == 4 {
|
||||
limit += 120
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) {
|
||||
var total_reps_zero uint = 0
|
||||
var total_reps_non_zero uint = 0
|
||||
var count_reps_zero uint = 1
|
||||
var count_reps_non_zero uint = 1
|
||||
var i uint
|
||||
for i = 0; i < length; {
|
||||
var value byte = depth[i]
|
||||
var reps uint = 1
|
||||
var k uint
|
||||
for k = i + 1; k < length && depth[k] == value; k++ {
|
||||
reps++
|
||||
}
|
||||
|
||||
if reps >= 3 && value == 0 {
|
||||
total_reps_zero += reps
|
||||
count_reps_zero++
|
||||
}
|
||||
|
||||
if reps >= 4 && value != 0 {
|
||||
total_reps_non_zero += reps
|
||||
count_reps_non_zero++
|
||||
}
|
||||
|
||||
i += reps
|
||||
}
|
||||
|
||||
*use_rle_for_non_zero = total_reps_non_zero > count_reps_non_zero*2
|
||||
*use_rle_for_zero = total_reps_zero > count_reps_zero*2
|
||||
}
|
||||
|
||||
/* Write a Huffman tree from bit depths into the bit-stream representation
|
||||
of a Huffman tree. The generated Huffman tree is to be compressed once
|
||||
more using a Huffman tree */
|
||||
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
|
||||
var previous_value byte = initialRepeatedCodeLength
|
||||
var i uint
|
||||
var use_rle_for_non_zero bool = false
|
||||
var use_rle_for_zero bool = false
|
||||
var new_length uint = length
|
||||
/* Throw away trailing zeros. */
|
||||
for i = 0; i < length; i++ {
|
||||
if depth[length-i-1] == 0 {
|
||||
new_length--
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
/* First gather statistics on if it is a good idea to do RLE. */
|
||||
if length > 50 {
|
||||
/* Find RLE coding for longer codes.
|
||||
Shorter codes seem not to benefit from RLE. */
|
||||
decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero)
|
||||
}
|
||||
|
||||
/* Actual RLE coding. */
|
||||
for i = 0; i < new_length; {
|
||||
var value byte = depth[i]
|
||||
var reps uint = 1
|
||||
if (value != 0 && use_rle_for_non_zero) || (value == 0 && use_rle_for_zero) {
|
||||
var k uint
|
||||
for k = i + 1; k < new_length && depth[k] == value; k++ {
|
||||
reps++
|
||||
}
|
||||
}
|
||||
|
||||
if value == 0 {
|
||||
writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data)
|
||||
} else {
|
||||
writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data)
|
||||
previous_value = value
|
||||
}
|
||||
|
||||
i += reps
|
||||
}
|
||||
}
|
||||
|
||||
var reverseBits_kLut = [16]uint{
|
||||
0x00,
|
||||
0x08,
|
||||
0x04,
|
||||
0x0C,
|
||||
0x02,
|
||||
0x0A,
|
||||
0x06,
|
||||
0x0E,
|
||||
0x01,
|
||||
0x09,
|
||||
0x05,
|
||||
0x0D,
|
||||
0x03,
|
||||
0x0B,
|
||||
0x07,
|
||||
0x0F,
|
||||
}
|
||||
|
||||
func reverseBits(num_bits uint, bits uint16) uint16 {
|
||||
var retval uint = reverseBits_kLut[bits&0x0F]
|
||||
var i uint
|
||||
for i = 4; i < num_bits; i += 4 {
|
||||
retval <<= 4
|
||||
bits = uint16(bits >> 4)
|
||||
retval |= reverseBits_kLut[bits&0x0F]
|
||||
}
|
||||
|
||||
retval >>= ((0 - num_bits) & 0x03)
|
||||
return uint16(retval)
|
||||
}
|
||||
|
||||
/* 0..15 are values for bits */
|
||||
const maxHuffmanBits = 16
|
||||
|
||||
/* Get the actual bit values for a tree of bit depths. */
|
||||
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
|
||||
var bl_count = [maxHuffmanBits]uint16{0}
|
||||
var next_code [maxHuffmanBits]uint16
|
||||
var i uint
|
||||
/* In Brotli, all bit depths are [1..15]
|
||||
0 bit depth means that the symbol does not exist. */
|
||||
|
||||
var code int = 0
|
||||
for i = 0; i < len; i++ {
|
||||
bl_count[depth[i]]++
|
||||
}
|
||||
|
||||
bl_count[0] = 0
|
||||
next_code[0] = 0
|
||||
for i = 1; i < maxHuffmanBits; i++ {
|
||||
code = (code + int(bl_count[i-1])) << 1
|
||||
next_code[i] = uint16(code)
|
||||
}
|
||||
|
||||
for i = 0; i < len; i++ {
|
||||
if depth[i] != 0 {
|
||||
bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]])
|
||||
next_code[depth[i]]++
|
||||
}
|
||||
}
|
||||
}
|
||||
4394
vendor/github.com/andybalholm/brotli/entropy_encode_static.go
generated
vendored
Normal file
4394
vendor/github.com/andybalholm/brotli/entropy_encode_static.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
290
vendor/github.com/andybalholm/brotli/fast_log.go
generated
vendored
Normal file
290
vendor/github.com/andybalholm/brotli/fast_log.go
generated
vendored
Normal file
@ -0,0 +1,290 @@
|
||||
package brotli
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Utilities for fast computation of logarithms. */
|
||||
|
||||
func log2FloorNonZero(n uint) uint32 {
|
||||
return uint32(bits.Len(n)) - 1
|
||||
}
|
||||
|
||||
/* A lookup table for small values of log2(int) to be used in entropy
|
||||
computation.
|
||||
|
||||
", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */
|
||||
var kLog2Table = []float32{
|
||||
0.0000000000000000,
|
||||
0.0000000000000000,
|
||||
1.0000000000000000,
|
||||
1.5849625007211563,
|
||||
2.0000000000000000,
|
||||
2.3219280948873622,
|
||||
2.5849625007211561,
|
||||
2.8073549220576042,
|
||||
3.0000000000000000,
|
||||
3.1699250014423126,
|
||||
3.3219280948873626,
|
||||
3.4594316186372978,
|
||||
3.5849625007211565,
|
||||
3.7004397181410922,
|
||||
3.8073549220576037,
|
||||
3.9068905956085187,
|
||||
4.0000000000000000,
|
||||
4.0874628412503400,
|
||||
4.1699250014423122,
|
||||
4.2479275134435852,
|
||||
4.3219280948873626,
|
||||
4.3923174227787607,
|
||||
4.4594316186372973,
|
||||
4.5235619560570131,
|
||||
4.5849625007211570,
|
||||
4.6438561897747244,
|
||||
4.7004397181410926,
|
||||
4.7548875021634691,
|
||||
4.8073549220576037,
|
||||
4.8579809951275728,
|
||||
4.9068905956085187,
|
||||
4.9541963103868758,
|
||||
5.0000000000000000,
|
||||
5.0443941193584534,
|
||||
5.0874628412503400,
|
||||
5.1292830169449664,
|
||||
5.1699250014423122,
|
||||
5.2094533656289501,
|
||||
5.2479275134435852,
|
||||
5.2854022188622487,
|
||||
5.3219280948873626,
|
||||
5.3575520046180838,
|
||||
5.3923174227787607,
|
||||
5.4262647547020979,
|
||||
5.4594316186372973,
|
||||
5.4918530963296748,
|
||||
5.5235619560570131,
|
||||
5.5545888516776376,
|
||||
5.5849625007211570,
|
||||
5.6147098441152083,
|
||||
5.6438561897747244,
|
||||
5.6724253419714961,
|
||||
5.7004397181410926,
|
||||
5.7279204545631996,
|
||||
5.7548875021634691,
|
||||
5.7813597135246599,
|
||||
5.8073549220576046,
|
||||
5.8328900141647422,
|
||||
5.8579809951275719,
|
||||
5.8826430493618416,
|
||||
5.9068905956085187,
|
||||
5.9307373375628867,
|
||||
5.9541963103868758,
|
||||
5.9772799234999168,
|
||||
6.0000000000000000,
|
||||
6.0223678130284544,
|
||||
6.0443941193584534,
|
||||
6.0660891904577721,
|
||||
6.0874628412503400,
|
||||
6.1085244567781700,
|
||||
6.1292830169449672,
|
||||
6.1497471195046822,
|
||||
6.1699250014423122,
|
||||
6.1898245588800176,
|
||||
6.2094533656289510,
|
||||
6.2288186904958804,
|
||||
6.2479275134435861,
|
||||
6.2667865406949019,
|
||||
6.2854022188622487,
|
||||
6.3037807481771031,
|
||||
6.3219280948873617,
|
||||
6.3398500028846252,
|
||||
6.3575520046180847,
|
||||
6.3750394313469254,
|
||||
6.3923174227787598,
|
||||
6.4093909361377026,
|
||||
6.4262647547020979,
|
||||
6.4429434958487288,
|
||||
6.4594316186372982,
|
||||
6.4757334309663976,
|
||||
6.4918530963296748,
|
||||
6.5077946401986964,
|
||||
6.5235619560570131,
|
||||
6.5391588111080319,
|
||||
6.5545888516776376,
|
||||
6.5698556083309478,
|
||||
6.5849625007211561,
|
||||
6.5999128421871278,
|
||||
6.6147098441152092,
|
||||
6.6293566200796095,
|
||||
6.6438561897747253,
|
||||
6.6582114827517955,
|
||||
6.6724253419714952,
|
||||
6.6865005271832185,
|
||||
6.7004397181410917,
|
||||
6.7142455176661224,
|
||||
6.7279204545631988,
|
||||
6.7414669864011465,
|
||||
6.7548875021634691,
|
||||
6.7681843247769260,
|
||||
6.7813597135246599,
|
||||
6.7944158663501062,
|
||||
6.8073549220576037,
|
||||
6.8201789624151887,
|
||||
6.8328900141647422,
|
||||
6.8454900509443757,
|
||||
6.8579809951275719,
|
||||
6.8703647195834048,
|
||||
6.8826430493618416,
|
||||
6.8948177633079437,
|
||||
6.9068905956085187,
|
||||
6.9188632372745955,
|
||||
6.9307373375628867,
|
||||
6.9425145053392399,
|
||||
6.9541963103868758,
|
||||
6.9657842846620879,
|
||||
6.9772799234999168,
|
||||
6.9886846867721664,
|
||||
7.0000000000000000,
|
||||
7.0112272554232540,
|
||||
7.0223678130284544,
|
||||
7.0334230015374501,
|
||||
7.0443941193584534,
|
||||
7.0552824355011898,
|
||||
7.0660891904577721,
|
||||
7.0768155970508317,
|
||||
7.0874628412503400,
|
||||
7.0980320829605272,
|
||||
7.1085244567781700,
|
||||
7.1189410727235076,
|
||||
7.1292830169449664,
|
||||
7.1395513523987937,
|
||||
7.1497471195046822,
|
||||
7.1598713367783891,
|
||||
7.1699250014423130,
|
||||
7.1799090900149345,
|
||||
7.1898245588800176,
|
||||
7.1996723448363644,
|
||||
7.2094533656289492,
|
||||
7.2191685204621621,
|
||||
7.2288186904958804,
|
||||
7.2384047393250794,
|
||||
7.2479275134435861,
|
||||
7.2573878426926521,
|
||||
7.2667865406949019,
|
||||
7.2761244052742384,
|
||||
7.2854022188622487,
|
||||
7.2946207488916270,
|
||||
7.3037807481771031,
|
||||
7.3128829552843557,
|
||||
7.3219280948873617,
|
||||
7.3309168781146177,
|
||||
7.3398500028846243,
|
||||
7.3487281542310781,
|
||||
7.3575520046180847,
|
||||
7.3663222142458151,
|
||||
7.3750394313469254,
|
||||
7.3837042924740528,
|
||||
7.3923174227787607,
|
||||
7.4008794362821844,
|
||||
7.4093909361377026,
|
||||
7.4178525148858991,
|
||||
7.4262647547020979,
|
||||
7.4346282276367255,
|
||||
7.4429434958487288,
|
||||
7.4512111118323299,
|
||||
7.4594316186372973,
|
||||
7.4676055500829976,
|
||||
7.4757334309663976,
|
||||
7.4838157772642564,
|
||||
7.4918530963296748,
|
||||
7.4998458870832057,
|
||||
7.5077946401986964,
|
||||
7.5156998382840436,
|
||||
7.5235619560570131,
|
||||
7.5313814605163119,
|
||||
7.5391588111080319,
|
||||
7.5468944598876373,
|
||||
7.5545888516776376,
|
||||
7.5622424242210728,
|
||||
7.5698556083309478,
|
||||
7.5774288280357487,
|
||||
7.5849625007211561,
|
||||
7.5924570372680806,
|
||||
7.5999128421871278,
|
||||
7.6073303137496113,
|
||||
7.6147098441152075,
|
||||
7.6220518194563764,
|
||||
7.6293566200796095,
|
||||
7.6366246205436488,
|
||||
7.6438561897747244,
|
||||
7.6510516911789290,
|
||||
7.6582114827517955,
|
||||
7.6653359171851765,
|
||||
7.6724253419714952,
|
||||
7.6794800995054464,
|
||||
7.6865005271832185,
|
||||
7.6934869574993252,
|
||||
7.7004397181410926,
|
||||
7.7073591320808825,
|
||||
7.7142455176661224,
|
||||
7.7210991887071856,
|
||||
7.7279204545631996,
|
||||
7.7347096202258392,
|
||||
7.7414669864011465,
|
||||
7.7481928495894596,
|
||||
7.7548875021634691,
|
||||
7.7615512324444795,
|
||||
7.7681843247769260,
|
||||
7.7747870596011737,
|
||||
7.7813597135246608,
|
||||
7.7879025593914317,
|
||||
7.7944158663501062,
|
||||
7.8008998999203047,
|
||||
7.8073549220576037,
|
||||
7.8137811912170374,
|
||||
7.8201789624151887,
|
||||
7.8265484872909159,
|
||||
7.8328900141647422,
|
||||
7.8392037880969445,
|
||||
7.8454900509443757,
|
||||
7.8517490414160571,
|
||||
7.8579809951275719,
|
||||
7.8641861446542798,
|
||||
7.8703647195834048,
|
||||
7.8765169465650002,
|
||||
7.8826430493618425,
|
||||
7.8887432488982601,
|
||||
7.8948177633079446,
|
||||
7.9008668079807496,
|
||||
7.9068905956085187,
|
||||
7.9128893362299619,
|
||||
7.9188632372745955,
|
||||
7.9248125036057813,
|
||||
7.9307373375628867,
|
||||
7.9366379390025719,
|
||||
7.9425145053392399,
|
||||
7.9483672315846778,
|
||||
7.9541963103868758,
|
||||
7.9600019320680806,
|
||||
7.9657842846620870,
|
||||
7.9715435539507720,
|
||||
7.9772799234999168,
|
||||
7.9829935746943104,
|
||||
7.9886846867721664,
|
||||
7.9943534368588578,
|
||||
}
|
||||
|
||||
/* Faster logarithm for small integers, with the property of log2(0) == 0. */
|
||||
func fastLog2(v uint) float64 {
|
||||
if v < uint(len(kLog2Table)) {
|
||||
return float64(kLog2Table[v])
|
||||
}
|
||||
|
||||
return math.Log2(float64(v))
|
||||
}
|
||||
45
vendor/github.com/andybalholm/brotli/find_match_length.go
generated
vendored
Normal file
45
vendor/github.com/andybalholm/brotli/find_match_length.go
generated
vendored
Normal file
@ -0,0 +1,45 @@
|
||||
package brotli
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math/bits"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Function to find maximal matching prefixes of strings. */
|
||||
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
|
||||
var matched uint = 0
|
||||
_, _ = s1[limit-1], s2[limit-1] // bounds check
|
||||
switch runtime.GOARCH {
|
||||
case "amd64":
|
||||
// Compare 8 bytes at at time.
|
||||
for matched+8 <= limit {
|
||||
w1 := binary.LittleEndian.Uint64(s1[matched:])
|
||||
w2 := binary.LittleEndian.Uint64(s2[matched:])
|
||||
if w1 != w2 {
|
||||
return matched + uint(bits.TrailingZeros64(w1^w2)>>3)
|
||||
}
|
||||
matched += 8
|
||||
}
|
||||
case "386":
|
||||
// Compare 4 bytes at at time.
|
||||
for matched+4 <= limit {
|
||||
w1 := binary.LittleEndian.Uint32(s1[matched:])
|
||||
w2 := binary.LittleEndian.Uint32(s2[matched:])
|
||||
if w1 != w2 {
|
||||
return matched + uint(bits.TrailingZeros32(w1^w2)>>3)
|
||||
}
|
||||
matched += 4
|
||||
}
|
||||
}
|
||||
for matched < limit && s1[matched] == s2[matched] {
|
||||
matched++
|
||||
}
|
||||
return matched
|
||||
}
|
||||
287
vendor/github.com/andybalholm/brotli/h10.go
generated
vendored
Normal file
287
vendor/github.com/andybalholm/brotli/h10.go
generated
vendored
Normal file
@ -0,0 +1,287 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
func (*h10) HashTypeLength() uint {
|
||||
return 4
|
||||
}
|
||||
|
||||
func (*h10) StoreLookahead() uint {
|
||||
return 128
|
||||
}
|
||||
|
||||
func hashBytesH10(data []byte) uint32 {
|
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
||||
|
||||
/* The higher bits contain more mixture from the multiplication,
|
||||
so we take our results from there. */
|
||||
return h >> (32 - 17)
|
||||
}
|
||||
|
||||
/* A (forgetful) hash table where each hash bucket contains a binary tree of
|
||||
sequences whose first 4 bytes share the same hash code.
|
||||
Each sequence is 128 long and is identified by its starting
|
||||
position in the input data. The binary tree is sorted by the lexicographic
|
||||
order of the sequences, and it is also a max-heap with respect to the
|
||||
starting positions. */
|
||||
type h10 struct {
|
||||
hasherCommon
|
||||
window_mask_ uint
|
||||
buckets_ [1 << 17]uint32
|
||||
invalid_pos_ uint32
|
||||
forest []uint32
|
||||
}
|
||||
|
||||
func (h *h10) Initialize(params *encoderParams) {
|
||||
h.window_mask_ = (1 << params.lgwin) - 1
|
||||
h.invalid_pos_ = uint32(0 - h.window_mask_)
|
||||
var num_nodes uint = uint(1) << params.lgwin
|
||||
h.forest = make([]uint32, 2*num_nodes)
|
||||
}
|
||||
|
||||
func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) {
|
||||
var invalid_pos uint32 = h.invalid_pos_
|
||||
var i uint32
|
||||
for i = 0; i < 1<<17; i++ {
|
||||
h.buckets_[i] = invalid_pos
|
||||
}
|
||||
}
|
||||
|
||||
func leftChildIndexH10(self *h10, pos uint) uint {
|
||||
return 2 * (pos & self.window_mask_)
|
||||
}
|
||||
|
||||
func rightChildIndexH10(self *h10, pos uint) uint {
|
||||
return 2*(pos&self.window_mask_) + 1
|
||||
}
|
||||
|
||||
/* Stores the hash of the next 4 bytes and in a single tree-traversal, the
|
||||
hash bucket's binary tree is searched for matches and is re-rooted at the
|
||||
current position.
|
||||
|
||||
If less than 128 data is available, the hash bucket of the
|
||||
current position is searched for matches, but the state of the hash table
|
||||
is not changed, since we can not know the final sorting order of the
|
||||
current (incomplete) sequence.
|
||||
|
||||
This function must be called with increasing cur_ix positions. */
|
||||
func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch {
|
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
||||
var max_comp_len uint = brotli_min_size_t(max_length, 128)
|
||||
var should_reroot_tree bool = (max_length >= 128)
|
||||
var key uint32 = hashBytesH10(data[cur_ix_masked:])
|
||||
var forest []uint32 = self.forest
|
||||
var prev_ix uint = uint(self.buckets_[key])
|
||||
var node_left uint = leftChildIndexH10(self, cur_ix)
|
||||
var node_right uint = rightChildIndexH10(self, cur_ix)
|
||||
var best_len_left uint = 0
|
||||
var best_len_right uint = 0
|
||||
var depth_remaining uint
|
||||
/* The forest index of the rightmost node of the left subtree of the new
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
|
||||
/* The forest index of the leftmost node of the right subtree of the new
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
|
||||
/* The match length of the rightmost node of the left subtree of the new
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
|
||||
/* The match length of the leftmost node of the right subtree of the new
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
if should_reroot_tree {
|
||||
self.buckets_[key] = uint32(cur_ix)
|
||||
}
|
||||
|
||||
for depth_remaining = 64; ; depth_remaining-- {
|
||||
var backward uint = cur_ix - prev_ix
|
||||
var prev_ix_masked uint = prev_ix & ring_buffer_mask
|
||||
if backward == 0 || backward > max_backward || depth_remaining == 0 {
|
||||
if should_reroot_tree {
|
||||
forest[node_left] = self.invalid_pos_
|
||||
forest[node_right] = self.invalid_pos_
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
{
|
||||
var cur_len uint = brotli_min_size_t(best_len_left, best_len_right)
|
||||
var len uint
|
||||
assert(cur_len <= 128)
|
||||
len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len)
|
||||
if matches != nil && len > *best_len {
|
||||
*best_len = uint(len)
|
||||
initBackwardMatch(&matches[0], backward, uint(len))
|
||||
matches = matches[1:]
|
||||
}
|
||||
|
||||
if len >= max_comp_len {
|
||||
if should_reroot_tree {
|
||||
forest[node_left] = forest[leftChildIndexH10(self, prev_ix)]
|
||||
forest[node_right] = forest[rightChildIndexH10(self, prev_ix)]
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if data[cur_ix_masked+len] > data[prev_ix_masked+len] {
|
||||
best_len_left = uint(len)
|
||||
if should_reroot_tree {
|
||||
forest[node_left] = uint32(prev_ix)
|
||||
}
|
||||
|
||||
node_left = rightChildIndexH10(self, prev_ix)
|
||||
prev_ix = uint(forest[node_left])
|
||||
} else {
|
||||
best_len_right = uint(len)
|
||||
if should_reroot_tree {
|
||||
forest[node_right] = uint32(prev_ix)
|
||||
}
|
||||
|
||||
node_right = leftChildIndexH10(self, prev_ix)
|
||||
prev_ix = uint(forest[node_right])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the
|
||||
length of max_length and stores the position cur_ix in the hash table.
|
||||
|
||||
Sets *num_matches to the number of matches found, and stores the found
|
||||
matches in matches[0] to matches[*num_matches - 1]. The matches will be
|
||||
sorted by strictly increasing length and (non-strictly) increasing
|
||||
distance. */
|
||||
func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint {
|
||||
var orig_matches []backwardMatch = matches
|
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
||||
var best_len uint = 1
|
||||
var short_match_max_backward uint
|
||||
if params.quality != hqZopflificationQuality {
|
||||
short_match_max_backward = 16
|
||||
} else {
|
||||
short_match_max_backward = 64
|
||||
}
|
||||
var stop uint = cur_ix - short_match_max_backward
|
||||
var dict_matches [maxStaticDictionaryMatchLen + 1]uint32
|
||||
var i uint
|
||||
if cur_ix < short_match_max_backward {
|
||||
stop = 0
|
||||
}
|
||||
for i = cur_ix - 1; i > stop && best_len <= 2; i-- {
|
||||
var prev_ix uint = i
|
||||
var backward uint = cur_ix - prev_ix
|
||||
if backward > max_backward {
|
||||
break
|
||||
}
|
||||
|
||||
prev_ix &= ring_buffer_mask
|
||||
if data[cur_ix_masked] != data[prev_ix] || data[cur_ix_masked+1] != data[prev_ix+1] {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
||||
if len > best_len {
|
||||
best_len = uint(len)
|
||||
initBackwardMatch(&matches[0], backward, uint(len))
|
||||
matches = matches[1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if best_len < max_length {
|
||||
matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches)
|
||||
}
|
||||
|
||||
for i = 0; i <= maxStaticDictionaryMatchLen; i++ {
|
||||
dict_matches[i] = kInvalidMatch
|
||||
}
|
||||
{
|
||||
var minlen uint = brotli_max_size_t(4, best_len+1)
|
||||
if findAllStaticDictionaryMatches(dictionary, data[cur_ix_masked:], minlen, max_length, dict_matches[0:]) {
|
||||
var maxlen uint = brotli_min_size_t(maxStaticDictionaryMatchLen, max_length)
|
||||
var l uint
|
||||
for l = minlen; l <= maxlen; l++ {
|
||||
var dict_id uint32 = dict_matches[l]
|
||||
if dict_id < kInvalidMatch {
|
||||
var distance uint = max_backward + gap + uint(dict_id>>5) + 1
|
||||
if distance <= params.dist.max_distance {
|
||||
initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31))
|
||||
matches = matches[1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return uint(-cap(matches) + cap(orig_matches))
|
||||
}
|
||||
|
||||
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
|
||||
current sequence, without returning any matches.
|
||||
REQUIRES: ix + 128 <= end-of-current-block */
|
||||
func (h *h10) Store(data []byte, mask uint, ix uint) {
|
||||
var max_backward uint = h.window_mask_ - windowGap + 1
|
||||
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
|
||||
storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil)
|
||||
}
|
||||
|
||||
func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
||||
var i uint = ix_start
|
||||
var j uint = ix_start
|
||||
if ix_start+63 <= ix_end {
|
||||
i = ix_end - 63
|
||||
}
|
||||
|
||||
if ix_start+512 <= i {
|
||||
for ; j < i; j += 8 {
|
||||
h.Store(data, mask, j)
|
||||
}
|
||||
}
|
||||
|
||||
for ; i < ix_end; i++ {
|
||||
h.Store(data, mask, i)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 128 {
|
||||
var i_start uint = position - 128 + 1
|
||||
var i_end uint = brotli_min_size_t(position, i_start+num_bytes)
|
||||
/* Store the last `128 - 1` positions in the hasher.
|
||||
These could not be calculated before, since they require knowledge
|
||||
of both the previous and the current block. */
|
||||
|
||||
var i uint
|
||||
for i = i_start; i < i_end; i++ {
|
||||
/* Maximum distance is window size - 16, see section 9.1. of the spec.
|
||||
Furthermore, we have to make sure that we don't look further back
|
||||
from the start of the next block than the window size, otherwise we
|
||||
could access already overwritten areas of the ring-buffer. */
|
||||
var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i)
|
||||
|
||||
/* We know that i + 128 <= position + num_bytes, i.e. the
|
||||
end of the current block and that we have at least
|
||||
128 tail in the ring-buffer. */
|
||||
storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
|
||||
const maxNumMatchesH10 = 128
|
||||
|
||||
func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
func (*h10) PrepareDistanceCache(distance_cache []int) {
|
||||
panic("unimplemented")
|
||||
}
|
||||
214
vendor/github.com/andybalholm/brotli/h5.go
generated
vendored
Normal file
214
vendor/github.com/andybalholm/brotli/h5.go
generated
vendored
Normal file
@ -0,0 +1,214 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to
|
||||
help create backward references to previous data.
|
||||
|
||||
This is a hash map of fixed size (bucket_size_) to a ring buffer of
|
||||
fixed size (block_size_). The ring buffer contains the last block_size_
|
||||
index positions of the given hash key in the compressed data. */
|
||||
func (*h5) HashTypeLength() uint {
|
||||
return 4
|
||||
}
|
||||
|
||||
func (*h5) StoreLookahead() uint {
|
||||
return 4
|
||||
}
|
||||
|
||||
/* HashBytes is the function that chooses the bucket to place the address in. */
|
||||
func hashBytesH5(data []byte, shift int) uint32 {
|
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
||||
|
||||
/* The higher bits contain more mixture from the multiplication,
|
||||
so we take our results from there. */
|
||||
return uint32(h >> uint(shift))
|
||||
}
|
||||
|
||||
type h5 struct {
|
||||
hasherCommon
|
||||
bucket_size_ uint
|
||||
block_size_ uint
|
||||
hash_shift_ int
|
||||
block_mask_ uint32
|
||||
num []uint16
|
||||
buckets []uint32
|
||||
}
|
||||
|
||||
func (h *h5) Initialize(params *encoderParams) {
|
||||
h.hash_shift_ = 32 - h.params.bucket_bits
|
||||
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
|
||||
h.block_size_ = uint(1) << uint(h.params.block_bits)
|
||||
h.block_mask_ = uint32(h.block_size_ - 1)
|
||||
h.num = make([]uint16, h.bucket_size_)
|
||||
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
|
||||
}
|
||||
|
||||
func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) {
|
||||
var num []uint16 = h.num
|
||||
var partial_prepare_threshold uint = h.bucket_size_ >> 6
|
||||
/* Partial preparation is 100 times slower (per socket). */
|
||||
if one_shot && input_size <= partial_prepare_threshold {
|
||||
var i uint
|
||||
for i = 0; i < input_size; i++ {
|
||||
var key uint32 = hashBytesH5(data[i:], h.hash_shift_)
|
||||
num[key] = 0
|
||||
}
|
||||
} else {
|
||||
for i := 0; i < int(h.bucket_size_); i++ {
|
||||
num[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Look at 4 bytes at &data[ix & mask].
|
||||
Compute a hash from these, and store the value of ix at that position. */
|
||||
func (h *h5) Store(data []byte, mask uint, ix uint) {
|
||||
var num []uint16 = h.num
|
||||
var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_)
|
||||
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
|
||||
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
|
||||
h.buckets[offset] = uint32(ix)
|
||||
num[key]++
|
||||
}
|
||||
|
||||
func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
||||
var i uint
|
||||
for i = ix_start; i < ix_end; i++ {
|
||||
h.Store(data, mask, i)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
||||
/* Prepare the hashes for three last bytes of the last write.
|
||||
These could not be calculated before, since they require knowledge
|
||||
of both the previous and the current block. */
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-3)
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-2)
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-1)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *h5) PrepareDistanceCache(distance_cache []int) {
|
||||
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
|
||||
}
|
||||
|
||||
/* Find a longest backward match of &data[cur_ix] up to the length of
|
||||
max_length and stores the position cur_ix in the hash table.
|
||||
|
||||
REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache
|
||||
values; if this method is invoked repeatedly with the same distance
|
||||
cache values, it is enough to invoke PrepareDistanceCacheH5 once.
|
||||
|
||||
Does not look for matches longer than max_length.
|
||||
Does not look for matches further away than max_backward.
|
||||
Writes the best match into |out|.
|
||||
|out|->score is updated only if a better match is found. */
|
||||
func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
||||
var num []uint16 = h.num
|
||||
var buckets []uint32 = h.buckets
|
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
||||
var min_score uint = out.score
|
||||
var best_score uint = out.score
|
||||
var best_len uint = out.len
|
||||
var i uint
|
||||
var bucket []uint32
|
||||
/* Don't accept a short copy from far away. */
|
||||
out.len = 0
|
||||
|
||||
out.len_code_delta = 0
|
||||
|
||||
/* Try last distance first. */
|
||||
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
|
||||
var backward uint = uint(distance_cache[i])
|
||||
var prev_ix uint = uint(cur_ix - backward)
|
||||
if prev_ix >= cur_ix {
|
||||
continue
|
||||
}
|
||||
|
||||
if backward > max_backward {
|
||||
continue
|
||||
}
|
||||
|
||||
prev_ix &= ring_buffer_mask
|
||||
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
||||
if len >= 3 || (len == 2 && i < 2) {
|
||||
/* Comparing for >= 2 does not change the semantics, but just saves for
|
||||
a few unnecessary binary logarithms in backward reference score,
|
||||
since we are not interested in such short matches. */
|
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
||||
if best_score < score {
|
||||
if i != 0 {
|
||||
score -= backwardReferencePenaltyUsingLastDistance(i)
|
||||
}
|
||||
if best_score < score {
|
||||
best_score = score
|
||||
best_len = uint(len)
|
||||
out.len = best_len
|
||||
out.distance = backward
|
||||
out.score = best_score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_)
|
||||
bucket = buckets[key<<uint(h.params.block_bits):]
|
||||
var down uint
|
||||
if uint(num[key]) > h.block_size_ {
|
||||
down = uint(num[key]) - h.block_size_
|
||||
} else {
|
||||
down = 0
|
||||
}
|
||||
for i = uint(num[key]); i > down; {
|
||||
var prev_ix uint
|
||||
i--
|
||||
prev_ix = uint(bucket[uint32(i)&h.block_mask_])
|
||||
var backward uint = cur_ix - prev_ix
|
||||
if backward > max_backward {
|
||||
break
|
||||
}
|
||||
|
||||
prev_ix &= ring_buffer_mask
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
||||
if len >= 4 {
|
||||
/* Comparing for >= 3 does not change the semantics, but just saves
|
||||
for a few unnecessary binary logarithms in backward reference
|
||||
score, since we are not interested in such short matches. */
|
||||
var score uint = backwardReferenceScore(uint(len), backward)
|
||||
if best_score < score {
|
||||
best_score = score
|
||||
best_len = uint(len)
|
||||
out.len = best_len
|
||||
out.distance = backward
|
||||
out.score = best_score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
|
||||
num[key]++
|
||||
}
|
||||
|
||||
if min_score == out.score {
|
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
|
||||
}
|
||||
}
|
||||
216
vendor/github.com/andybalholm/brotli/h6.go
generated
vendored
Normal file
216
vendor/github.com/andybalholm/brotli/h6.go
generated
vendored
Normal file
@ -0,0 +1,216 @@
|
||||
package brotli
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to
|
||||
help create backward references to previous data.
|
||||
|
||||
This is a hash map of fixed size (bucket_size_) to a ring buffer of
|
||||
fixed size (block_size_). The ring buffer contains the last block_size_
|
||||
index positions of the given hash key in the compressed data. */
|
||||
func (*h6) HashTypeLength() uint {
|
||||
return 8
|
||||
}
|
||||
|
||||
func (*h6) StoreLookahead() uint {
|
||||
return 8
|
||||
}
|
||||
|
||||
/* HashBytes is the function that chooses the bucket to place the address in. */
|
||||
func hashBytesH6(data []byte, mask uint64, shift int) uint32 {
|
||||
var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long
|
||||
|
||||
/* The higher bits contain more mixture from the multiplication,
|
||||
so we take our results from there. */
|
||||
return uint32(h >> uint(shift))
|
||||
}
|
||||
|
||||
type h6 struct {
|
||||
hasherCommon
|
||||
bucket_size_ uint
|
||||
block_size_ uint
|
||||
hash_shift_ int
|
||||
hash_mask_ uint64
|
||||
block_mask_ uint32
|
||||
num []uint16
|
||||
buckets []uint32
|
||||
}
|
||||
|
||||
func (h *h6) Initialize(params *encoderParams) {
|
||||
h.hash_shift_ = 64 - h.params.bucket_bits
|
||||
h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len)
|
||||
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
|
||||
h.block_size_ = uint(1) << uint(h.params.block_bits)
|
||||
h.block_mask_ = uint32(h.block_size_ - 1)
|
||||
h.num = make([]uint16, h.bucket_size_)
|
||||
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
|
||||
}
|
||||
|
||||
func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) {
|
||||
var num []uint16 = h.num
|
||||
var partial_prepare_threshold uint = h.bucket_size_ >> 6
|
||||
/* Partial preparation is 100 times slower (per socket). */
|
||||
if one_shot && input_size <= partial_prepare_threshold {
|
||||
var i uint
|
||||
for i = 0; i < input_size; i++ {
|
||||
var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
|
||||
num[key] = 0
|
||||
}
|
||||
} else {
|
||||
for i := 0; i < int(h.bucket_size_); i++ {
|
||||
num[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Look at 4 bytes at &data[ix & mask].
|
||||
Compute a hash from these, and store the value of ix at that position. */
|
||||
func (h *h6) Store(data []byte, mask uint, ix uint) {
|
||||
var num []uint16 = h.num
|
||||
var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
|
||||
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
|
||||
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
|
||||
h.buckets[offset] = uint32(ix)
|
||||
num[key]++
|
||||
}
|
||||
|
||||
func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
||||
var i uint
|
||||
for i = ix_start; i < ix_end; i++ {
|
||||
h.Store(data, mask, i)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
|
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
|
||||
/* Prepare the hashes for three last bytes of the last write.
|
||||
These could not be calculated before, since they require knowledge
|
||||
of both the previous and the current block. */
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-3)
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-2)
|
||||
h.Store(ringbuffer, ringbuffer_mask, position-1)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *h6) PrepareDistanceCache(distance_cache []int) {
|
||||
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
|
||||
}
|
||||
|
||||
/* Find a longest backward match of &data[cur_ix] up to the length of
|
||||
max_length and stores the position cur_ix in the hash table.
|
||||
|
||||
REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache
|
||||
values; if this method is invoked repeatedly with the same distance
|
||||
cache values, it is enough to invoke PrepareDistanceCacheH6 once.
|
||||
|
||||
Does not look for matches longer than max_length.
|
||||
Does not look for matches further away than max_backward.
|
||||
Writes the best match into |out|.
|
||||
|out|->score is updated only if a better match is found. */
|
||||
func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
||||
var num []uint16 = h.num
|
||||
var buckets []uint32 = h.buckets
|
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask
|
||||
var min_score uint = out.score
|
||||
var best_score uint = out.score
|
||||
var best_len uint = out.len
|
||||
var i uint
|
||||
var bucket []uint32
|
||||
/* Don't accept a short copy from far away. */
|
||||
out.len = 0
|
||||
|
||||
out.len_code_delta = 0
|
||||
|
||||
/* Try last distance first. */
|
||||
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
|
||||
var backward uint = uint(distance_cache[i])
|
||||
var prev_ix uint = uint(cur_ix - backward)
|
||||
if prev_ix >= cur_ix {
|
||||
continue
|
||||
}
|
||||
|
||||
if backward > max_backward {
|
||||
continue
|
||||
}
|
||||
|
||||
prev_ix &= ring_buffer_mask
|
||||
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
||||
if len >= 3 || (len == 2 && i < 2) {
|
||||
/* Comparing for >= 2 does not change the semantics, but just saves for
|
||||
a few unnecessary binary logarithms in backward reference score,
|
||||
since we are not interested in such short matches. */
|
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
|
||||
if best_score < score {
|
||||
if i != 0 {
|
||||
score -= backwardReferencePenaltyUsingLastDistance(i)
|
||||
}
|
||||
if best_score < score {
|
||||
best_score = score
|
||||
best_len = uint(len)
|
||||
out.len = best_len
|
||||
out.distance = backward
|
||||
out.score = best_score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
|
||||
bucket = buckets[key<<uint(h.params.block_bits):]
|
||||
var down uint
|
||||
if uint(num[key]) > h.block_size_ {
|
||||
down = uint(num[key]) - h.block_size_
|
||||
} else {
|
||||
down = 0
|
||||
}
|
||||
for i = uint(num[key]); i > down; {
|
||||
var prev_ix uint
|
||||
i--
|
||||
prev_ix = uint(bucket[uint32(i)&h.block_mask_])
|
||||
var backward uint = cur_ix - prev_ix
|
||||
if backward > max_backward {
|
||||
break
|
||||
}
|
||||
|
||||
prev_ix &= ring_buffer_mask
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
|
||||
continue
|
||||
}
|
||||
{
|
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
|
||||
if len >= 4 {
|
||||
/* Comparing for >= 3 does not change the semantics, but just saves
|
||||
for a few unnecessary binary logarithms in backward reference
|
||||
score, since we are not interested in such short matches. */
|
||||
var score uint = backwardReferenceScore(uint(len), backward)
|
||||
if best_score < score {
|
||||
best_score = score
|
||||
best_len = uint(len)
|
||||
out.len = best_len
|
||||
out.distance = backward
|
||||
out.score = best_score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
|
||||
num[key]++
|
||||
}
|
||||
|
||||
if min_score == out.score {
|
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user