diff --git a/Makefile b/Makefile
index 7db3259..2ca6827 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ clean:
@find . -name ".DS_Store" -exec rm -v {} \;
docker-pull:
+ docker-compose --project-name ${PROJECT_NAME} -f deployments/docker-compose-test.yaml build --pull
docker-compose --project-name ${PROJECT_NAME} -f deployments/docker-compose-test.yaml pull
docker-restart: docker-stop docker-start
diff --git a/cmd/relay/action/server/start.go b/cmd/relay/action/server/start.go
index 450d9b1..64bf8f0 100644
--- a/cmd/relay/action/server/start.go
+++ b/cmd/relay/action/server/start.go
@@ -15,7 +15,7 @@ import (
"git.ptzo.gdn/feditools/relay/internal/language"
"git.ptzo.gdn/feditools/relay/internal/logic/logic1"
"git.ptzo.gdn/feditools/relay/internal/metrics"
- "git.ptzo.gdn/feditools/relay/internal/runner/faktory"
+ "git.ptzo.gdn/feditools/relay/internal/runner/asynq"
"git.ptzo.gdn/feditools/relay/internal/scheduler"
"git.ptzo.gdn/feditools/relay/internal/token"
"github.com/spf13/viper"
@@ -160,15 +160,31 @@ var Start action.Action = func(topCtx context.Context) error {
logicMod.SetNotifier(notifier)
// create runner
- runnerMod, err := faktory.New(logicMod)
+ l.Debug("creating runner module")
+ runnerAddr := viper.GetString(config.Keys.RedisAddress)
+ runnerPassword := viper.GetString(config.Keys.RedisPassword)
+ runnerDB := viper.GetInt(config.Keys.RedisDB)
+
+ if viper.GetString(config.Keys.RunnerAddress) != "" {
+ runnerAddr = viper.GetString(config.Keys.RunnerAddress)
+ runnerPassword = viper.GetString(config.Keys.RunnerPassword)
+ runnerDB = viper.GetInt(config.Keys.RunnerDB)
+ }
+
+ runnerMod, err := asynq.New(&asynq.Config{
+ Logic: logicMod,
+
+ Address: runnerAddr,
+ Password: runnerPassword,
+ DB: runnerDB,
+ })
if err != nil {
- l.Errorf("runner: %s", err.Error())
+ l.Errorf("runner server: %s", err.Error())
cancel()
return err
}
logicMod.SetRunner(runnerMod)
- runnerMod.Start(ctx)
// create scheduler
schedulerMod, err := scheduler.New(logicMod, runnerMod)
diff --git a/cmd/relay/action/worker/notifier.go b/cmd/relay/action/worker/notifier.go
new file mode 100644
index 0000000..d8e891a
--- /dev/null
+++ b/cmd/relay/action/worker/notifier.go
@@ -0,0 +1,30 @@
+package worker
+
+import (
+ "git.ptzo.gdn/feditools/relay/internal/logic"
+ "git.ptzo.gdn/feditools/relay/internal/notification/manager"
+ "git.ptzo.gdn/feditools/relay/internal/notification/telegram"
+)
+
+func newNotifier(
+ logicMod logic.Logic,
+) (*manager.Manager, error) {
+ l := logger.WithField("func", "newNotifier")
+
+ newManager, err := manager.New()
+ if err != nil {
+ l.Errorf("notification manager: %s", err.Error())
+
+ return nil, err
+ }
+
+ telegramMod, err := telegram.New(logicMod)
+ if err != nil {
+ l.Errorf("notification telegram: %s", err.Error())
+
+ return nil, err
+ }
+ newManager.AddService(telegramMod)
+
+ return newManager, nil
+}
diff --git a/cmd/relay/action/worker/start.go b/cmd/relay/action/worker/start.go
index 53fe03e..127d535 100644
--- a/cmd/relay/action/worker/start.go
+++ b/cmd/relay/action/worker/start.go
@@ -12,7 +12,7 @@ import (
"git.ptzo.gdn/feditools/relay/internal/kv/redis"
"git.ptzo.gdn/feditools/relay/internal/logic/logic1"
"git.ptzo.gdn/feditools/relay/internal/metrics"
- "git.ptzo.gdn/feditools/relay/internal/runner/faktory"
+ "git.ptzo.gdn/feditools/relay/internal/runner/asynq"
"git.ptzo.gdn/feditools/relay/internal/token"
"github.com/spf13/viper"
"github.com/uptrace/uptrace-go/uptrace"
@@ -127,15 +127,50 @@ var Start action.Action = func(ctx context.Context) error {
}
// create runner
- runnerMod, err := faktory.New(logicMod)
+ // create runner
+ l.Debug("creating runner module")
+ runnerAddr := viper.GetString(config.Keys.RedisAddress)
+ runnerPassword := viper.GetString(config.Keys.RedisPassword)
+ runnerDB := viper.GetInt(config.Keys.RedisDB)
+
+ if viper.GetString(config.Keys.RunnerAddress) != "" {
+ runnerAddr = viper.GetString(config.Keys.RunnerAddress)
+ runnerPassword = viper.GetString(config.Keys.RunnerPassword)
+ runnerDB = viper.GetInt(config.Keys.RunnerDB)
+ }
+
+ runnerMod, err := asynq.New(&asynq.Config{
+ Logic: logicMod,
+
+ Concurrency: viper.GetInt(config.Keys.RunnerConcurrency),
+ Address: runnerAddr,
+ Password: runnerPassword,
+ DB: runnerDB,
+ })
if err != nil {
- l.Errorf("runner: %s", err.Error())
+ l.Errorf("runner server: %s", err.Error())
cancel()
return err
}
+ defer func() {
+ l.Debug("closing runner")
+ err := runnerMod.Stop()
+ if err != nil {
+ l.Errorf("closing runner: %s", err.Error())
+ }
+ }()
logicMod.SetRunner(runnerMod)
- runnerMod.Start(ctx)
+
+ // create language module
+ notifier, err := newNotifier(logicMod)
+ if err != nil {
+ l.Errorf("notifier: %s", err.Error())
+ cancel()
+
+ return err
+ }
+ logicMod.SetNotifier(notifier)
// ** start application **
errChan := make(chan error)
@@ -153,6 +188,15 @@ var Start action.Action = func(ctx context.Context) error {
}
}(metricsServer, errChan)
+ // start runner server
+ go func(m *asynq.Runner, errChan chan error) {
+ l.Debug("starting runner server")
+ err := m.Start(ctx)
+ if err != nil {
+ errChan <- fmt.Errorf("runner server: %s", err.Error())
+ }
+ }(runnerMod, errChan)
+
// wait for event
select {
case sig := <-stopSigChan:
diff --git a/cmd/relay/flag/runner.go b/cmd/relay/flag/runner.go
index 0983dff..7ab11f0 100644
--- a/cmd/relay/flag/runner.go
+++ b/cmd/relay/flag/runner.go
@@ -9,4 +9,7 @@ import (
func Runner(cmd *cobra.Command, values config.Values) {
cmd.PersistentFlags().Int(config.Keys.RunnerConcurrency, values.RunnerConcurrency, usage.RunnerConcurrency)
cmd.PersistentFlags().Int(config.Keys.RunnerPoolSize, values.RunnerPoolSize, usage.RunnerPoolSize)
+ cmd.PersistentFlags().String(config.Keys.RunnerAddress, values.RunnerAddress, usage.RunnerAddress)
+ cmd.PersistentFlags().Int(config.Keys.RunnerDB, values.RunnerDB, usage.RunnerDB)
+ cmd.PersistentFlags().String(config.Keys.RunnerPassword, values.RunnerPassword, usage.RunnerPassword)
}
diff --git a/deployments/docker-compose-test.yaml b/deployments/docker-compose-test.yaml
index fe83911..f14168c 100644
--- a/deployments/docker-compose-test.yaml
+++ b/deployments/docker-compose-test.yaml
@@ -2,26 +2,52 @@
version: '3'
services:
postgres:
- image: postgres:14
+ image: postgres:15
ports:
- - 127.0.0.1:5432:5432/tcp
+ - "127.0.0.1:5432:5432/tcp"
environment:
- POSTGRES_PASSWORD=test
- POSTGRES_USER=test
- POSTGRES_DB=test
restart: always
- faktory:
- image: contribsys/faktory:1.6.1
- command: /faktory -b :7419 -w :7420 -e production
- ports:
- - 127.0.0.1:7419:7419/tcp
- - 127.0.0.1:7420:7420/tcp
- environment:
- - FAKTORY_PASSWORD=test
- restart: always
+# faktory:
+# image: contribsys/faktory:1.6.1
+# command: /faktory -b :7419 -w :7420 -e production
+# ports:
+# - 127.0.0.1:7419:7419/tcp
+# - 127.0.0.1:7420:7420/tcp
+# environment:
+# - FAKTORY_PASSWORD=test
+# restart: always
redis:
image: redis:6
command: redis-server --requirepass test
ports:
- "127.0.0.1:6379:6379/tcp"
restart: always
+ redis-runner:
+ image: redis:6
+ command: redis-server --requirepass test2
+ ports:
+ - "127.0.0.1:6380:6379/tcp"
+ restart: always
+
+ asynqmon:
+ image: hibiken/asynqmon:latest
+ ports:
+ - "127.0.0.1:8080:8080/tcp"
+ environment:
+ - "REDIS_ADDR=redis-runner:6379"
+ - "REDIS_PASSWORD=test2"
+ - "ENABLE_METRICS_EXPORTER=true"
+ - "PROMETHEUS_ADDR=http://prometheus:9090"
+ prometheus:
+ build:
+ context: ./prometheus
+ ports:
+ - "127.0.0.1:9090:9090/tcp"
+ grafana:
+ build:
+ context: ./grafana
+ ports:
+ - "127.0.0.1:3000:3000/tcp"
\ No newline at end of file
diff --git a/deployments/grafana/Dockerfile b/deployments/grafana/Dockerfile
new file mode 100644
index 0000000..f588f05
--- /dev/null
+++ b/deployments/grafana/Dockerfile
@@ -0,0 +1,4 @@
+FROM grafana/grafana:latest
+
+ADD grafana.ini /etc/grafana/
+ADD datasource.yaml /etc/grafana/provisioning/datasources/
diff --git a/deployments/grafana/datasource.yaml b/deployments/grafana/datasource.yaml
new file mode 100644
index 0000000..f88db84
--- /dev/null
+++ b/deployments/grafana/datasource.yaml
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+ - name: Prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus:9090
+ isDefault: true
+ editable: true
\ No newline at end of file
diff --git a/deployments/grafana/grafana.ini b/deployments/grafana/grafana.ini
new file mode 100644
index 0000000..9cee64c
--- /dev/null
+++ b/deployments/grafana/grafana.ini
@@ -0,0 +1,1334 @@
+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+;app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+;instance_name = ${HOSTNAME}
+
+# force migration will run migrations that might cause dataloss
+;force_migration = false
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+;data = /var/lib/grafana
+
+# Temporary files in `data` directory older than given duration will be removed
+;temp_data_lifetime = 24h
+
+# Directory where grafana can store logs
+;logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+;plugins = /var/lib/grafana/plugins
+
+# folder that contains provisioning config files that grafana will apply on startup and while running.
+;provisioning = conf/provisioning
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, h2, socket)
+;protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port to use
+;http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+;root_url = %(protocol)s://%(domain)s:%(http_port)s/
+
+# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
+;serve_from_sub_path = false
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket gid
+# Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner
+# It is recommended to set the gid as http server user gid
+# Not set when the value is -1
+;socket_gid =
+
+# Unix socket mode
+;socket_mode =
+
+# Unix socket path
+;socket =
+
+# CDN Url
+;cdn_url =
+
+# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections.
+# `0` means there is no timeout for reading the request.
+;read_timeout = 0
+
+# This setting enables you to specify additional headers that the server adds to HTTP(S) responses.
+[server.custom_response_headers]
+#exampleHeader1 = exampleValue1
+#exampleHeader2 = exampleValue2
+
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+;type = sqlite3
+;host = 127.0.0.1:3306
+;name = grafana
+;user = root
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+
+# Use either URL or the previous fields to configure the database
+# Example: mysql://user:secret@host:port/database
+;url =
+
+# For "postgres", use either "disable", "require" or "verify-full"
+# For "mysql", use either "true", "false", or "skip-verify".
+;ssl_mode = disable
+
+# Database drivers may support different transaction isolation levels.
+# Currently, only "mysql" driver supports isolation levels.
+# If the value is empty - driver's default isolation level is applied.
+# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE".
+;isolation_level =
+
+;ca_cert_path =
+;client_key_path =
+;client_cert_path =
+;server_cert_name =
+
+# For "sqlite3" only, path relative to data_path setting
+;path = grafana.db
+
+# Max idle conn setting default is 2
+;max_idle_conn = 2
+
+# Max conn setting default is 0 (mean not set)
+;max_open_conn =
+
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
+# Set to true to log the sql calls and execution times.
+;log_queries =
+
+# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
+;cache_mode = private
+
+# For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false.
+;wal = false
+
+# For "mysql" only if migrationLocking feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0.
+;locking_attempt_timeout_sec = 0
+
+# For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled).
+;query_retries = 0
+
+# For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5.
+;transaction_retries = 5
+
+################################### Data sources #########################
+[datasources]
+# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API.
+;datasource_limit = 5000
+
+#################################### Cache server #############################
+[remote_cache]
+# Either "redis", "memcached" or "database" default is "database"
+;type = database
+
+# cache connectionstring options
+# database: will use Grafana primary database.
+# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'.
+# memcache: 127.0.0.1:11211
+;connstr =
+
+# prefix prepended to all the keys in the remote cache
+; prefix =
+
+# This enables encryption of values stored in the remote cache
+;encryption =
+
+#################################### Data proxy ###########################
+[dataproxy]
+
+# This enables data proxy logging, default is false
+;logging = false
+
+# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds.
+# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set.
+;timeout = 30
+
+# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds.
+;dialTimeout = 10
+
+# How many seconds the data proxy waits before sending a keepalive probe request.
+;keep_alive_seconds = 30
+
+# How many seconds the data proxy waits for a successful TLS Handshake before timing out.
+;tls_handshake_timeout_seconds = 10
+
+# How many seconds the data proxy will wait for a server's first response headers after
+# fully writing the request headers if the request has an "Expect: 100-continue"
+# header. A value of 0 will result in the body being sent immediately, without
+# waiting for the server to approve.
+;expect_continue_timeout_seconds = 1
+
+# Optionally limits the total number of connections per host, including connections in the dialing,
+# active, and idle states. On limit violation, dials will block.
+# A value of zero (0) means no limit.
+;max_conns_per_host = 0
+
+# The maximum number of idle connections that Grafana will keep alive.
+;max_idle_connections = 100
+
+# How many seconds the data proxy keeps an idle connection open before timing out.
+;idle_conn_timeout_seconds = 90
+
+# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
+;send_user_header = false
+
+# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests.
+;response_limit = 0
+
+# Limits the number of rows that Grafana will process from SQL data sources.
+;row_limit = 1000000
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+;reporting_enabled = true
+
+# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs
+;reporting_distributor = grafana-labs
+
+# Set to false to disable all checks to https://grafana.com
+# for new versions of grafana. The check is used
+# in some UI views to notify that a grafana update exists.
+# This option does not cause any auto updates, nor send any information
+# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version.
+;check_for_updates = true
+
+# Set to false to disable all checks to https://grafana.com
+# for new versions of plugins. The check is used
+# in some UI views to notify that a plugin update exists.
+# This option does not cause any auto updates, nor send any information
+# only a GET request to https://grafana.com to get the latest versions.
+;check_for_plugin_updates = true
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+# Google Analytics 4 tracking code, only enabled if you specify an id here
+;google_analytics_4_id =
+
+# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc.
+;google_analytics_4_send_manual_page_views = false
+
+# Google Tag Manager ID, only enabled if you specify an id here
+;google_tag_manager_id =
+
+# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set
+;rudderstack_write_key =
+
+# Rudderstack data plane url, enabled only if rudderstack_write_key is also set
+;rudderstack_data_plane_url =
+
+# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set
+;rudderstack_sdk_url =
+
+# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config
+;rudderstack_config_url =
+
+# Controls if the UI contains any links to user feedback forms
+;feedback_links_enabled = true
+
+#################################### Security ####################################
+[security]
+# disable creation of admin user on first start of grafana
+;disable_initial_admin_creation = false
+
+# default admin user, created on startup
+;admin_user = admin
+
+# default admin password, can be changed before first start of grafana, or in profile settings
+;admin_password = admin
+
+# default admin email, created on startup
+;admin_email = admin@localhost
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# current key provider used for envelope encryption, default to static value specified by secret_key
+;encryption_provider = secretKey.v1
+
+# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1
+;available_encryption_providers =
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+# disable protection against brute force login attempts
+;disable_brute_force_login_protection = false
+
+# set to true if you host Grafana behind HTTPS. default is false.
+;cookie_secure = false
+
+# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled"
+;cookie_samesite = lax
+
+# set to true if you want to allow browsers to render Grafana in a ,