diff --git a/.config/example.yml b/.config/example.yml index 02c0296172..43f6d7e923 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -85,28 +85,6 @@ redis: #prefix: example-prefix #db: 1 -# Please configure either MeiliSearch *or* Sonic. -# If both MeiliSearch and Sonic configurations are present, MeiliSearch will take precedence. - -# ┌───────────────────────────┐ -#───┘ MeiliSearch configuration └───────────────────────────────────── -#meilisearch: -# host: meilisearch -# port: 7700 -# ssl: false -# apiKey: - -# ┌─────────────────────┐ -#───┘ Sonic configuration └───────────────────────────────────── - -#sonic: -# host: localhost -# port: 1491 -# auth: SecretPassword -# collection: notes -# bucket: default - - # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/README.md b/README.md index b7ba3cc6fe..6edbd5efbd 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,6 @@ If you have access to a server that supports one of the sources below, I recomme ### Optional dependencies - [FFmpeg](https://ffmpeg.org/) for video transcoding -- Full text search (one of the following) - - [Sonic](https://crates.io/crates/sonic-server) - - [MeiliSearch](https://www.meilisearch.com/) - - [ElasticSearch](https://www.elastic.co/elasticsearch/) - Caching server (one of the following) - [DragonflyDB](https://www.dragonflydb.io/) (recommended) - [KeyDB](https://keydb.dev/) diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 184c0366c9..d1037fd593 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -6,10 +6,12 @@ services: ports: - "26379:6379" db: - image: docker.io/postgres:16-alpine + image: docker.io/groonga/pgroonga:latest-alpine-16-slim environment: - "POSTGRES_PASSWORD=password" - "POSTGRES_USER=firefish" - "POSTGRES_DB=firefish_db" ports: - "25432:5432" + volumes: + - "./install.sql:/docker-entrypoint-initdb.d/install.sql:ro" diff --git a/dev/install.sql b/dev/install.sql new file mode 100644 index 0000000000..11418be32b --- /dev/null +++ b/dev/install.sql @@ -0,0 +1 @@ +CREATE EXTENSION pgroonga; diff --git a/docker-compose.example.yml b/docker-compose.example.yml index a5fd4c315e..c2c60786d9 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -8,9 +8,6 @@ services: depends_on: - db - redis -### Uncomment one of the following to use a search engine -# - meilisearch -# - sonic ports: - "3000:3000" networks: @@ -34,7 +31,7 @@ services: db: restart: unless-stopped - image: docker.io/postgres:16-alpine + image: docker.io/groonga/pgroonga:latest-alpine-16-slim container_name: firefish_db networks: - calcnet @@ -43,33 +40,6 @@ services: volumes: - ./db:/var/lib/postgresql/data -### Only one of the below should be used. -### Meilisearch is better overall, but resource-intensive. Sonic is a very light full text search engine. - -# meilisearch: -# container_name: meilisearch -# image: getmeili/meilisearch:v1.1.1 -# environment: -# - MEILI_ENV=${MEILI_ENV:-development} -# ports: -# - "7700:7700" -# networks: -# - calcnet -# volumes: -# - ./meili_data:/meili_data -# restart: unless-stopped - -# sonic: -# restart: unless-stopped -# image: docker.io/valeriansaliou/sonic:v1.4.0 -# logging: -# driver: none -# networks: -# - calcnet -# volumes: -# - ./sonic:/var/lib/sonic/store -# - ./sonic/config.cfg:/etc/sonic.cfg - networks: calcnet: # web: diff --git a/docs/api-change.md b/docs/api-change.md index 4046e5c48c..cf9799f00b 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -2,6 +2,10 @@ Breaking changes are indicated by the :warning: icon. +## Unreleased + +- `admin/search/index-all` is removed since posts are now indexed automatically. + ## v20240301 - With the addition of new features, the following endpoints are added: diff --git a/docs/changelog.md b/docs/changelog.md index 71ba30951e..049e5e0ed2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ Critical security updates are indicated by the :warning: icon. +## Unreleased + +- Introduce new full-text search engine + ## v20240301 - Add a page (`/my/follow-requests/sent`) to check your follow requests that haven't been approved diff --git a/docs/downgrade.sql b/docs/downgrade.sql index 8a7769dcb9..b1fc95eab7 100644 --- a/docs/downgrade.sql +++ b/docs/downgrade.sql @@ -1,6 +1,7 @@ BEGIN; DELETE FROM "migrations" WHERE name IN ( + 'Pgroonga1698420787202', 'ChangeDefaultConfigs1709251460718', 'AddReplyMuting1704851359889', 'FixNoteUrlIndex1709129810501', @@ -12,6 +13,12 @@ DELETE FROM "migrations" WHERE name IN ( 'RemoveNativeUtilsMigration1705877093218' ); +-- pgroonga +DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"; +DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"; +DROP INDEX "IDX_fcb770976ff8240af5799e3ffc"; +DROP EXTENSION pgroonga CASCADE; + -- change-default-configs ALTER TABLE "user_profile" ALTER COLUMN "noCrawle" SET DEFAULT false; ALTER TABLE "user_profile" ALTER COLUMN "publicReactions" SET DEFAULT false; diff --git a/docs/notice-for-admins.md b/docs/notice-for-admins.md index 1edb5ecca2..276ffea7b8 100644 --- a/docs/notice-for-admins.md +++ b/docs/notice-for-admins.md @@ -1,3 +1,97 @@ +# Unreleased + +The full-text search engine in Firefish has been changed to [PGroonga](https://pgroonga.github.io/). This is no longer an optional feature, so please enable PGroonga on your system. If you are using Sonic, Meilisearch, or Elasticsearch, you can also uninstall it from your system and remove the settings from `.config/default.yml`. + +## For systemd/pm2 users + +### 1. Install PGroonga + +Please execute `psql --version` to check your PostgreSQL major version. This will print a message like this: + +```text +psql (PostgreSQL) 16.1 +``` + +In this case, your PostgreSQL major version is `16`. + +There are official installation instructions for many operating systems on , so please follow the instructions on this page. However, since many users are using Ubuntu, and there are no instructions for Arch Linux, we explicitly list the instructions for Ubuntu and Arch Linux here. Please keep in mind that this is not official information and the procedures may change. + +#### Ubuntu + +1. Add apt repository + ```sh + sudo apt install -y software-properties-common + sudo add-apt-repository -y universe + sudo add-apt-repository -y ppa:groonga/ppa + sudo apt install -y wget lsb-release + wget https://packages.groonga.org/ubuntu/groonga-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release --codename --short).deb + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release --codename --short)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt update + ``` +2. Install PGroonga + ```sh + # Please replace "16" with your PostgreSQL major version + sudo apt install postgresql-16-pgdg-pgroonga + ``` + +#### Arch Linux + +You can install PGroonga from the Arch User Repository. + +```sh +git clone https://aur.archlinux.org/pgroonga.git && cd pgroonga && makepkg -si +# or paru -S pgroonga +# or yay -S pgroonga +``` + +### 2. Enable PGroonga + +After the instllation, please execute this command to enable PGroonga: + +```sh +sudo --user=postgres psql --dbname=your_database_name --command='CREATE EXTENSION pgroonga;' +``` + +The database name can be found in `.config/default.yml`. +```yaml +db: + port: 5432 + db: database_name # substitute your_database_name with this + user: firefish + pass: password +``` + +## For Docker/Podman users + +Please edit your `docker-compose.yml` to replace the database container image from `docker.io/postgres` to `docker.io/groonga/pgroonga`. + +Please make sure to use the same PostgreSQL version. If you are using `docker.io/postgres:16-alpine` (PostgreSQL v16), the corresponding image tag is `docker.io/groonga/pgroonga:latest-alpine-16` (or `docker.io/groonga/pgroonga:latest-alpine-16-slim`). + +The list of tags can be found on . + +```yaml +db: + restart: unless-stopped + image: docker.io/groonga/pgroonga:latest-alpine-16-slim # change here + container_name: firefish_db +``` + +After that, execute this command to enable PGroonga: + +```sh +docker-compose up db --detach && docker-compose exec db sh -c 'psql --user="${POSTGRES_USER}" --dbname="${POSTGRES_DB}" --command="CREATE EXTENSION pgroonga;"' +# or podman-compose up db --detach && podman-compose exec db sh -c 'psql --user="${POSTGRES_USER}" --dbname="${POSTGRES_DB}" --command="CREATE EXTENSION pgroonga;"' +``` + +Once this is done, you can start Firefish as usual. + +```sh +docker pull registry.firefish.dev/firefish/firefish && docker-compose up --detach +# or podman pull registry.firefish.dev/firefish/firefish && podman-compose up --detach +``` + # v20240301 ## For all users diff --git a/locales/ca-ES.yml b/locales/ca-ES.yml index 7b66b3bd98..f0b3a9343d 100644 --- a/locales/ca-ES.yml +++ b/locales/ca-ES.yml @@ -1831,11 +1831,6 @@ pushNotificationAlreadySubscribed: Les notificacions push ja estan activades pushNotificationNotSupported: El vostre navegador o servidor no admet notificacions push license: Llicència -indexPosts: Índex de publicacions -indexFrom: Índex a partir de l'ID de Publicacions -indexFromDescription: Deixeu en blanc per indexar cada publicació -indexNotice: Ara indexant. Això probablement trigarà una estona, si us plau, no reinicieu - el servidor durant almenys una hora. _instanceTicker: none: No mostrar mai remote: Mostra per a usuaris remots diff --git a/locales/de-DE.yml b/locales/de-DE.yml index 53024bf845..42228695a4 100644 --- a/locales/de-DE.yml +++ b/locales/de-DE.yml @@ -1500,9 +1500,6 @@ _widgets: chooseList: Wählen Sie eine Liste aus userList: Benutzerliste serverInfo: Server-Infos - meiliStatus: Server-Status - meiliSize: Indexgröße - meiliIndexCount: Indexierte Beiträge _cw: hide: "Verbergen" show: "Inhalt anzeigen" @@ -2083,7 +2080,6 @@ preventAiLearning: KI gestütztes bot-scraping unterdrücken preventAiLearningDescription: Fordern Sie KI-Sprachmodelle von Drittanbietern auf, die von Ihnen hochgeladenen Inhalte, wie z. B. Beiträge und Bilder, nicht zu untersuchen. license: Lizenz -indexPosts: Gelistete Beiträge migrationConfirm: "Sind Sie absolut sicher, dass Sie Ihr Nutzerkonto zu diesem {account} umziehen möchten? Sobald Sie dies bestätigt haben, kann dies nicht mehr rückgängig gemacht werden und Ihr Nutzerkonto kann nicht mehr von ihnen genutzt werden.\nStellen @@ -2111,9 +2107,6 @@ _experiments: kann es zu Verlangsamungen beim Laden während des Imports kommen. noGraze: Bitte deaktivieren Sie die Browsererweiterung "Graze for Mastodon", da sie die Funktion von Firefish stört. -indexFrom: Indexieren ab Beitragskennung aufwärts -indexNotice: Wird jetzt indexiert. Dies wird wahrscheinlich eine Weile dauern, bitte - starten Sie Ihren Server für mindestens eine Stunde nicht neu. customKaTeXMacroDescription: "Richten Sie Makros ein, um mathematische Ausdrücke einfach zu schreiben! Die Notation entspricht den LaTeX-Befehlsdefinitionen und wird als \\newcommand{\\name}{content} oder \\newcommand{\\name}[number of arguments]{content} @@ -2132,7 +2125,6 @@ expandOnNoteClick: Beitrag bei Klick öffnen image: Bild video: Video audio: Audio -indexFromDescription: Leer lassen, um jeden Beitrag zu indexieren _filters: fromUser: Von Benutzer notesAfter: Beiträge nach diff --git a/locales/en-US.yml b/locales/en-US.yml index 0c40a340fb..476b340492 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1092,11 +1092,6 @@ migrationConfirm: "Are you absolutely sure you want to migrate your account to { as the account you're moving from." defaultReaction: "Default emoji reaction for outgoing and incoming posts" license: "License" -indexPosts: "Index Posts" -indexFrom: "Index from Post ID onwards" -indexFromDescription: "Leave blank to index every post" -indexNotice: "Now indexing. This will probably take a while, please don't restart - your server for at least an hour." customKaTeXMacro: "Custom KaTeX macros" customKaTeXMacroDescription: "Set up macros to write mathematical expressions easily! The notation conforms to the LaTeX command definitions and is written as \\newcommand{\\ @@ -1690,9 +1685,6 @@ _widgets: serverInfo: "Server Info" _userList: chooseList: "Select a list" - meiliStatus: "Server Status" - meiliSize: "Index size" - meiliIndexCount: "Indexed posts" _cw: hide: "Hide" diff --git a/locales/es-ES.yml b/locales/es-ES.yml index 328e53a10b..b0b1fc28e1 100644 --- a/locales/es-ES.yml +++ b/locales/es-ES.yml @@ -1474,9 +1474,6 @@ _widgets: _userList: chooseList: Seleccione una lista serverInfo: Información del servidor - meiliStatus: Estado del servidor - meiliSize: Tamaño del índice - meiliIndexCount: Publicaciones indizadas _cw: hide: "Ocultar" show: "Ver más" @@ -2123,13 +2120,10 @@ moveFromDescription: 'Esto pondrá un alias en tu cuenta antigua para así poder ingresa la etiqueta de la cuenta con el formato siguiente: @persona@servidor.tld' defaultReaction: Emoji por defecto para reaccionar a las publicaciones entrantes y salientes -indexFromDescription: Deja en blanco para indizar todas las publicaciones deletePasskeys: Borrar claves de paso deletePasskeysConfirm: Esto borrará irreversiblemente todas las claves de paso y de seguridad en esta cuenta, ¿Proceder? inputNotMatch: Las entradas no coinciden -indexFrom: Indizar desde la ID de la publicación en adelante -indexPosts: Indizar publicaciones isModerator: Moderador isAdmin: Administrador isPatron: Mecenas de Firefish @@ -2139,8 +2133,6 @@ migrationConfirm: "¿Estás absolutamente seguro de que quieres migrar a tu cuen {account}? Una vez hecho esto, no podrás revertir el cambio, ni tampoco usar tu cuenta normalmente.\nTambién, asegúrate de que has configurado ésta cuenta como la cuenta desde la cual estás migrando." -indexNotice: Indizando ahora. Esto puede llevar bastante tiempo, por favor, no reinicies - el servidor por lo menos hasta dentro de una hora. customKaTeXMacro: Macros KaTeX personalizadas customKaTeXMacroDescription: '¡Configura macros para escribir expresiones matemáticas fácilmente! La notación es conforme la las definiciones de comandos LaTeX y puede diff --git a/locales/fr-FR.yml b/locales/fr-FR.yml index 0b1043790c..a2500f7798 100644 --- a/locales/fr-FR.yml +++ b/locales/fr-FR.yml @@ -1400,10 +1400,7 @@ _widgets: _userList: chooseList: Sélectionner une liste unixClock: Horloge UNIX - meiliIndexCount: Publications indexées serverInfo: Info serveur - meiliStatus: État du serveur - meiliSize: Taille de l'index instanceCloud: Nuage de serveurs rssTicker: Bandeau RSS _cw: @@ -2061,9 +2058,6 @@ moveToLabel: 'Compte vers lequel vous migrez :' moveFrom: Migrer vers ce compte depuis un ancien compte defaultReaction: Émoji de réaction par défaut pour les publications entrantes et sortantes license: Licence -indexPosts: Indexer les publications -indexNotice: Indexation en cours. Cela prendra certainement du temps, veuillez ne - pas redémarrer votre serveur pour au moins une heure. customKaTeXMacro: Macros KaTeX personnalisées enableCustomKaTeXMacro: Activer les macros KaTeX personnalisées noteId: ID des publications @@ -2110,7 +2104,6 @@ expandOnNoteClick: Ouvrir la publications en cliquant preventAiLearning: Empêcher le récupération de données par des IA listsDesc: Les listes vous laissent créer des fils personnalisés avec des utilisateur·rice·s spécifié·e·s. Elles sont accessibles depuis la page des fils. -indexFromDescription: Laisser vide pour indexer toutes les publications _feeds: jsonFeed: flux JSON atom: Atom @@ -2120,7 +2113,6 @@ alt: ALT swipeOnMobile: Permettre le balayage entre les pages expandOnNoteClickDesc: Si désactivé, vous pourrez toujours ouvrir les publications dans le menu du clic droit et en cliquant sur l'horodatage. -indexFrom: Indexer à partir de l'ID des publications older: ancien newer: récent accessibility: Accessibilité diff --git a/locales/id-ID.yml b/locales/id-ID.yml index ee902f2be3..e3dffa689e 100644 --- a/locales/id-ID.yml +++ b/locales/id-ID.yml @@ -1390,14 +1390,11 @@ _widgets: aiscript: "Konsol AiScript" aichan: "Ai" rssTicker: Telegraf RSS - meiliIndexCount: Postingan yang terindeks userList: Daftar Pengguna instanceCloud: Server Awan unixClock: Jam UNIX - meiliSize: Ukuran indeks _userList: chooseList: Pilih daftar - meiliStatus: Status Server serverInfo: Info Server _cw: hide: "Sembunyikan" @@ -1984,8 +1981,6 @@ speed: Kecepatan slow: Pelan remoteOnly: Jarak jauh saja moveFrom: Dari akun lama pindahkan ke akun ini -indexNotice: Sedang mengindeks. Ini memerlukan beberapa waktu, mohon jangan mulai - ulang server setidaknya satu jam. sendPushNotificationReadMessage: Hapus pemberitahuan dorong saat pemberitahuan atau pesan relevan sudah dibaca moveAccountDescription: Proses ini permanen. Pastikan kamu sudah mengatur alias dari @@ -2010,7 +2005,6 @@ showAds: Tampilkan spanduk komunitas enterSendsMessage: Tekan Enter pada Pesan untuk mengirim pesan (matikan dengan Ctrl + Enter) showAdminUpdates: Indikasi versi Firefish baru tersedia (hanya admin) -indexFrom: Indeks dari Post ID berikutnya noteId: ID Postingan findOtherInstance: Cari server lain caption: Deskripsi itomatis @@ -2022,7 +2016,6 @@ moveFromDescription: Ini akan mengatur alias akun lamamu jadi kamu dapat pindah akun tersebut ke akun sekarang. Lakukan ini SEBELUM memindahkan akun lama. Silakan masukkan tag akun dengan format seperti @orang@server.com defaultReaction: Reaksi emoji bawaan untuk postingan keluar dan masuk -indexPosts: Indeks Postingan preventAiLearning: Cegah scraping bot AI customKaTeXMacro: Makro KaTeX khusus sendPushNotificationReadMessageCaption: Pemberitahuan yang berisi teks "{emptyPushNotificationMessage}" @@ -2052,7 +2045,6 @@ migrationConfirm: "Kamu sangat yakin ingin memindahkan akunmu ke {account}? Seka lagi secara normal. \nDan juga, harap pastikan kamu sudah mengatur akun sekarang sebagai akun yang dipindahkan." license: Lisensi -indexFromDescription: Kosongkan untuk mengindeks setiap postingan noGraze: Harap nonaktifkan ekstensi peramban "Graze for Mastodon", karena akan menganggu Firefish. silencedWarning: Halaman ini tampil karena pengguna ini datang dari server yang dibisukan diff --git a/locales/it-IT.yml b/locales/it-IT.yml index 7c3f130928..48242989da 100644 --- a/locales/it-IT.yml +++ b/locales/it-IT.yml @@ -1324,9 +1324,6 @@ _widgets: instanceCloud: Cloud del server unixClock: Orologio UNIX serverInfo: Informazioni sul server - meiliIndexCount: Post indicizzati - meiliStatus: Stato del server - meiliSize: Dimensione indice userList: Elenco utenti _cw: hide: "Nascondi" @@ -1838,9 +1835,7 @@ customSplashIconsDescription: Elenco degli URL di icone personalizzate da mostra le immagini siano su un URL statico, preferibilmente di dimensioni 192x192. swipeOnDesktop: Permetti lo swipe su desktop simile alla versione mobile logoImageUrl: URL del logo -indexFrom: Indicizza dal post ID customKaTeXMacro: Macro KaTeX personalizzate -indexPosts: Crea indice dei post signupsDisabled: Le iscrizioni su questo server al momento non sono possibili, ma puoi sempre iscriverti su un altro server! Se invece hai un codice di invito per questo server, inseriscilo qua sotto. @@ -1919,9 +1914,6 @@ lastActiveDate: Ultimo utilizzo enterSendsMessage: Premi "Invio" nei messaggi per inviare (altrimenti è "Ctrl + Invio") customMOTD: Messaggi di caricamento personalizzati (splash screen) replayTutorial: Ripeti il tutorial -indexFromDescription: Lascia vuoto per indicizzare tutti i post -indexNotice: Creazione indice in corso. Sarà necessario del tempo, fai attenzione - a non riavviare il server per almeno un'ora. enableCustomKaTeXMacro: Abilita le macro KaTeX personalizzate preventAiLearningDescription: Richiedi ai bot di intelligenza artificiale di terze parti di non studiare e acquisire il contenuto che carichi, come post e immagini. diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 1117231c18..64626b5fe9 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -974,10 +974,6 @@ migrationConfirm: "本当にこのアカウントを {account} に引っ越し この操作を行う前に引っ越し先のアカウントでエイリアスを作成する必要があります。エイリアスが作成されているか、必ず確認してください。" defaultReaction: "リモートとローカルの投稿に対するデフォルトの絵文字リアクション" license: "ライセンス" -indexPosts: "投稿をインデックス" -indexFrom: "この投稿ID以降をインデックスする" -indexFromDescription: "空白で全ての投稿を指定します" -indexNotice: "インデックスを開始しました。完了まで時間がかかる場合があるため、少なくとも1時間はサーバーを再起動しないでください。" customKaTeXMacro: "カスタムKaTeXマクロ" customKaTeXMacroDescription: "数式入力を楽にするためのマクロを設定しましょう！記法はLaTeXにおけるコマンドの定義と同様に \\newcommand{\\ name}{content} または \\newcommand{\\add}[2]{#1 + #2} のように記述します。後者の例では \\add{3}{foo} @@ -1453,10 +1449,7 @@ _widgets: userList: "ユーザーリスト" _userList: chooseList: "リストを選択" - meiliStatus: サーバーステータス serverInfo: サーバー情報 - meiliSize: インデックスサイズ - meiliIndexCount: インデックス済みの投稿 _cw: hide: "隠す" show: "もっと見る" diff --git a/locales/ko-KR.yml b/locales/ko-KR.yml index 722d48abf8..45a6c09c98 100644 --- a/locales/ko-KR.yml +++ b/locales/ko-KR.yml @@ -1331,10 +1331,7 @@ _widgets: serverInfo: 서버 정보 _userList: chooseList: 리스트 선택 - meiliStatus: 서버 정보 userList: 유저 목록 - meiliSize: 인덱스 크기 - meiliIndexCount: 인덱싱 완료된 게시물 rssTicker: RSS Ticker _cw: hide: "숨기기" @@ -1840,7 +1837,6 @@ customSplashIconsDescription: 유저가 페이지를 로딩/새로고침할 때 이미지는 되도록 정적 URL으로 구성하고, 192x192 해상도로 조정하여 주십시오. moveFromDescription: '이전 계정에 대한 별칭을 작성하여, 이 계정으로 옮길 수 있도록 합니다. 반드시 계정을 이전하기 전에 수행해야 합니다. 이전 계정을 다음과 같은 형식으로 입력하여 주십시오: @person@server.com' -indexFromDescription: 빈 칸으로 두면 모든 게시물을 인덱싱합니다 customKaTeXMacroDescription: 'KaTeX 매크로를 지정하여 수식을 더욱 편리하게 입력하세요! LaTeX의 커맨드 정의와 동일하게 \newcommand{\ 이름}{내용} 또는 \newcommand{\이름}[인수 갯수]{내용} 와 같이 입력하십시오. 예를 들어 \newcommand{\add}[2]{#1 + #2} 와 같이 정의한 경우 \add{3}{foo} 를 입력하면 3 + foo 으로 치환됩니다.매크로의 이름을 감싸는 중괄호를 소괄호() 또는 @@ -1895,7 +1891,6 @@ accessibility: 접근성 userSaysSomethingReasonReply: '{name} 님이 {reason} 을 포함하는 게시물에 답글했습니다' userSaysSomethingReasonRenote: '{name} 님이 {reason} 을 포함하는 게시물을 부스트했습니다' breakFollowConfirm: 팔로워를 해제하시겠습니까? -indexFrom: 이 게시물 ID부터 인덱싱하기 noThankYou: 괜찮습니다 hiddenTags: 숨길 해시태그 image: 이미지 @@ -1927,8 +1922,6 @@ removeMember: 멤버를 삭제 license: 라이선스 migrationConfirm: "정말로 이 계정을 {account}로 이사하시겠습니까? 한 번 이사하면, 현재 이 계정은 두 번 다시 사용할 수 없게 됩니다.\n또한, 이사 갈 계정에 현재 사용 중인 계정의 별칭을 올바르게 작성하였는지 다시 한 번 확인하십시오." -indexPosts: 게시물을 인덱싱 -indexNotice: 인덱싱을 시작했습니다. 이 작업은 시간이 많이 소요되므로, 최소 1시간 이내에 서버를 재시작하지 마십시오. noteId: 게시물 ID signupsDisabled: 현재 이 서버에서는 신규 등록을 받고 있지 않습니다. 초대 코드를 가지고 계신 경우 아래 칸에 입력해 주십시오. 초대 코드를 가지고 있지 않더라도, 신규 등록이 열려 있는 다른 서버에 등록하실 수 있습니다! diff --git a/locales/no-NO.yml b/locales/no-NO.yml index c817d46c24..d8b80e0e2c 100644 --- a/locales/no-NO.yml +++ b/locales/no-NO.yml @@ -939,7 +939,6 @@ allowedInstancesDescription: Tjenernavn for tjenere som skal hvitelistes. En per (Vil bare bli brukt i privat modus). previewNoteText: Forhåndsvisning recentNDays: Siste {n} dager -indexPosts: Indekser poster objectStorageUseProxy: Koble til gjennom en mellomtjener objectStorageUseProxyDesc: Skru av dette dersom du ikke vil bruke mellomtjenere for API-oppkoblinger @@ -1185,10 +1184,6 @@ moveFromDescription: Dette vil sette opp et alias for din gamle kontoen slik at kan flytte fra den gamle kontoen til denne. Gjør dette FØR du flytter fra den gamle kontoen. Skriv inn den gamle kontoen på formen @person@server.com defaultReaction: Standard emoji-reaksjon for utgående og innkommende poster -indexFrom: Indekser poster fra post-id og fremover -indexNotice: Indekserer. Dette vil sannsynligvis ta litt tid, ikke restart tjeneren - før det har gått minst en time. -indexFromDescription: La stå tom for å indeksere alle poster customKaTeXMacroDescription: 'Sett opp makroer for å skrive matematiske uttrykk enkelt. Notasjonen følger LaTeX-kommandoer og er skrevet som \newcommand{\ navn}{uttrykk} eller \newcommand{\navn}{antall argumenter}{uttrykk}. For eksempel vil \newcommand{\add}{2}{#1 @@ -1628,14 +1623,12 @@ _antennaSources: instances: Poster fra alle brukerne på denne tjeneren _widgets: timeline: Tidslinje - meiliSize: Indeks-størrelse instanceCloud: Tjenersky onlineUsers: Påloggede brukere clock: Klokke userList: Brukerliste rss: RSS-leser serverMetric: Tjenermetrikker - meiliIndexCount: Indekserte poster button: Knapp unixClock: Unix-klokke calendar: Kalender @@ -1647,7 +1640,6 @@ _widgets: photos: Bilder rssTicker: RSS-rulletekst aiscript: AiScript-konsoll - meiliStatus: Tjenerstatus memo: Notatlapp notifications: Varsler postForm: Ny post diff --git a/locales/pl-PL.yml b/locales/pl-PL.yml index 1369babee2..73b3939abb 100644 --- a/locales/pl-PL.yml +++ b/locales/pl-PL.yml @@ -1892,11 +1892,6 @@ sendPushNotificationReadMessageCaption: Powiadomienie zawierające tekst "{empty baterii Twojego urządzenia. defaultReaction: Domyślna reakcja emoji dla wychodzących i przychodzących wpisów license: Licencja -indexPosts: Indeksuj wpisy -indexFrom: Indeksuj wpisy od ID -indexFromDescription: Zostaw puste dla indeksowania wszystkich wpisów -indexNotice: Indeksuję. Zapewne zajmie to chwilę, nie restartuj serwera przez co najmniej - godzinę. customKaTeXMacro: Niestandardowe makra KaTeX enableCustomKaTeXMacro: Włącz niestandardowe makra KaTeX noteId: ID wpisu diff --git a/locales/ru-RU.yml b/locales/ru-RU.yml index 9de2c135b9..64742e5479 100644 --- a/locales/ru-RU.yml +++ b/locales/ru-RU.yml @@ -1375,9 +1375,6 @@ _widgets: userList: Список пользователей _userList: chooseList: Выберите список - meiliStatus: Состояние сервера - meiliSize: Размер индекса - meiliIndexCount: Индексированные посты serverInfo: Информация о сервере _cw: hide: "Спрятать" @@ -1951,11 +1948,6 @@ showUpdates: Показывать всплывающее окно при обн recommendedInstances: Рекомендованные серверы defaultReaction: Эмодзи реакция по умолчанию для выходящих и исходящих постов license: Лицензия -indexPosts: Индексировать посты -indexFrom: Индексировать начиная с идентификатора поста и далее -indexFromDescription: оставьте пустым для индексации каждого поста -indexNotice: Теперь индексирование. Вероятно, это займет некоторое время, пожалуйста, - не перезагружайте свой сервер по крайней мере в течение часа. customKaTeXMacro: Кастомные KaTex макросы enableCustomKaTeXMacro: Включить кастомные KaTeX макросы noteId: Идентификатор поста diff --git a/locales/th-TH.yml b/locales/th-TH.yml index bcd97e9f64..1302f86726 100644 --- a/locales/th-TH.yml +++ b/locales/th-TH.yml @@ -1316,8 +1316,6 @@ customMOTDDescription: ข้อความหน้าจอเริ่มต คั่นด้วยการขึ้นบรรทัดใหม่เพื่อแสดงแบบสุ่มทุกครั้งที่ผู้ใช้โหลดเว็บหรือโหลดหน้าเว็บซ้ำ caption: คำอธิบายโดยอัตโนมัติ moveToLabel: 'บัญชีที่คุณจะย้ายไปยัง:' -indexFromDescription: เว้นว่างไว้เพื่อสร้างดัชนีทุกโพสต์ -indexNotice: ตอนนี้กำลังจัดทำดัชนี การดำเนินการนี้อาจใช้เวลาสักครู่ โปรดอย่ารีสตาร์ทเซิร์ฟเวอร์เป็นเวลาอย่างน้อยหนึ่งชั่วโมง noteId: โพสต์ ID apps: แอป enableRecommendedTimeline: เปิดใช้งาน ไทม์ไลน์ที่แนะนำ @@ -1371,9 +1369,7 @@ moveFromDescription: การดำเนินการนี้จะตั migrationConfirm: "คุณแน่ใจหรือไม่ว่าคุณต้องการย้ายบัญชีของคุณไปยัง {account} เมื่อคุณทำเช่นนี้ คุณจะไม่สามารถกู้คืนมาได้ และคุณจะไม่สามารถใช้บัญชีของคุณได้ตามปกติอีก\nนอกจากนี้ โปรดตรวจสอบให้แน่ใจว่าคุณได้ตั้งบัญชีปัจจุบันนี้เป็นบัญชีที่คุณจะย้ายออก" -indexFrom: จัดทำดัชนีตั้งแต่ Post ID เป็นต้นไป license: ใบอนุญาต -indexPosts: ดัชนีโพสต์ signupsDisabled: การลงชื่อสมัครใช้บนเซิร์ฟเวอร์นี้ถูกปิดใช้งานอยู่ในขณะนี้ แต่คุณสามารถสมัครที่เซิร์ฟเวอร์อื่นได้ตลอดเวลา หากคุณมีรหัสเชิญสำหรับเซิร์ฟเวอร์นี้ โปรดป้อนรหัสด้านล่าง customKaTeXMacroDescription: 'ตั้งค่ามาโครเพื่อเขียนนิพจน์ทางคณิตศาสตร์ได้อย่างง่ายดาย diff --git a/locales/tr-TR.yml b/locales/tr-TR.yml index fc0c236e0a..16206614b3 100644 --- a/locales/tr-TR.yml +++ b/locales/tr-TR.yml @@ -158,7 +158,6 @@ _widgets: activity: Aktivite digitalClock: Dijital Saat unixClock: UNIX Saati - meiliIndexCount: Indexlenmiş gönderiler calendar: Takvim trends: Popüler memo: Yapışkan Notlar @@ -166,13 +165,11 @@ _widgets: federation: Federasyon instanceCloud: Sunucu Bulutu postForm: Gönderi Formu - meiliSize: Index boyutu slideshow: Slayt Gösterisi button: Düğme clock: Saat rss: RSS Okuyucu serverInfo: Sunucu Bilgisi - meiliStatus: Sunucu Durumu jobQueue: İş Sırası serverMetric: Sunucu Bilgileri _profile: @@ -473,7 +470,6 @@ activeEmailValidationDescription: Tek kullanımlık adreslerin kontrol edilmesi sağlar. İşaretlenmediğinde, yalnızca e-postanın biçimi doğrulanır. move: Taşı defaultReaction: Giden ve gelen gönderiler için varsayılan emoji tepkisi -indexPosts: Dizin Gönderileri youGotNewFollower: takip etti receiveFollowRequest: Takip isteği alındı followRequestAccepted: Takip isteği onaylandı @@ -1084,7 +1080,6 @@ check: Kontrol Et driveCapOverrideLabel: Bu kullanıcı için drive kapasitesini değiştirin numberOfPageCache: Önbelleğe alınan sayfa sayısı license: Lisans -indexFrom: Post ID'den itibaren dizin xl: XL notificationSetting: Bildirim ayarları fillAbuseReportDescription: Lütfen bu raporla ilgili ayrıntıları doldurun. Belirli @@ -1159,9 +1154,6 @@ migrationConfirm: "Hesabınızı {account} hesabına taşımak istediğinizden k emin misiniz? Bunu yaptığınızda, geri alamazsınız ve hesabınızı bir daha normal şekilde kullanamazsınız.\nAyrıca, lütfen bu cari hesabı, taşındığınız hesap olarak ayarladığınızdan emin olun." -indexFromDescription: Her gönderiyi dizine eklemek için boş bırakın -indexNotice: Şimdi indeksleniyor. Bu muhtemelen biraz zaman alacaktır, lütfen sunucunuzu - en az bir saat yeniden başlatmayın. customKaTeXMacro: Özel KaTeX makroları directNotes: Özel Mesajlar import: İçeri Aktar diff --git a/locales/uk-UA.yml b/locales/uk-UA.yml index 0905848e6a..f0b7296b78 100644 --- a/locales/uk-UA.yml +++ b/locales/uk-UA.yml @@ -1199,14 +1199,11 @@ _widgets: aiscript: "Консоль AiScript" _userList: chooseList: Оберіть список - meiliStatus: Стан сервера - meiliSize: Розмір індексу rssTicker: RSS-тікер instanceCloud: Хмара серверів unixClock: Годинник UNIX userList: Список користувачів serverInfo: Інформація про сервер - meiliIndexCount: Індексовані записи _cw: hide: "Сховати" show: "Показати більше" @@ -1977,11 +1974,6 @@ caption: Автоматичний опис showAdminUpdates: Вказати, що доступна нова версія Firefish (тільки для адміністратора) defaultReaction: Емодзі реакція за замовчуванням для вихідних і вхідних записів license: Ліцензія -indexPosts: Індексувати пости -indexFrom: Індексувати записи з ID -indexFromDescription: Залиште порожнім, щоб індексувати кожен запис -indexNotice: Зараз відбувається індексація. Це, ймовірно, займе деякий час, будь ласка, - не перезавантажуйте сервер принаймні годину. signupsDisabled: Реєстрація на цьому сервері наразі відключена, але ви завжди можете зареєструватися на іншому сервері! Якщо у вас є код запрошення на цей сервер, будь ласка, введіть його нижче. diff --git a/locales/vi-VN.yml b/locales/vi-VN.yml index 2c7790d7c8..119c745ac3 100644 --- a/locales/vi-VN.yml +++ b/locales/vi-VN.yml @@ -1437,9 +1437,6 @@ _widgets: userList: Danh sách người dùng _userList: chooseList: Chọn một danh sách - meiliSize: Kích cỡ chỉ mục - meiliIndexCount: Tút đã lập chỉ mục - meiliStatus: Trạng thái máy chủ serverInfo: Thông tin máy chủ _cw: hide: "Ẩn" @@ -2048,8 +2045,6 @@ hiddenTagsDescription: 'Liệt kê các hashtag (không có #) mà bạn muốn noInstances: Không có máy chủ nào manageGroups: Quản lý nhóm accessibility: Khả năng tiếp cận -indexNotice: Đang lập chỉ mục. Quá trình này có thể mất một lúc, vui lòng không khởi - động lại máy chủ của bạn sau ít nhất một giờ. breakFollowConfirm: Bạn có chắc muốn xóa người theo dõi? caption: Caption tự động objectStorageS3ForcePathStyle: Sử dụng URL điểm cuối dựa trên đường dẫn @@ -2069,7 +2064,6 @@ updateAvailable: Có bản cập nhật mới! swipeOnDesktop: Cho phép vuốt kiểu điện thoại trên máy tính moveFromLabel: 'Tài khoản cũ của bạn:' defaultReaction: Biểu cảm mặc định cho những tút đã đăng và sắp đăng -indexFromDescription: Để trống để lập chỉ mục toàn bộ donationLink: Liên kết tới trang tài trợ deletePasskeys: Xóa passkey delete2faConfirm: Thao tác này sẽ xóa 2FA trên tài khoản này một cách không thể phục @@ -2086,8 +2080,6 @@ audio: Âm thanh selectInstance: Chọn máy chủ userSaysSomethingReason: '{name} cho biết {reason}' pushNotification: Thông báo đẩy -indexPosts: Chỉ mục tút -indexFrom: Chỉ mục từ Post ID customKaTeXMacro: Tùy chỉnh macro KaTeX license: Giấy phép cw: Nội dung ẩn diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 1aa187e0a9..a53e0767e9 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -1350,9 +1350,6 @@ _widgets: aiscript: "AiScript 控制台" aichan: "小蓝" userList: 用户列表 - meiliStatus: 服务器状态 - meiliIndexCount: 已索引的帖子 - meiliSize: 索引大小 serverInfo: 服务器信息 _userList: chooseList: 选择列表 @@ -1917,7 +1914,6 @@ isAdmin: 管理员 findOtherInstance: 寻找其它服务器 moveFromDescription: 这将为您的旧账号设置一个别名，以便您可以从该旧账号迁移到当前账号。在从旧账号迁移之前执行此操作。请输入格式如 @person@server.com 的账号标签 -indexPosts: 索引帖子 signupsDisabled: 该服务器目前关闭注册，但您随时可以在另一台服务器上注册！如果您有该服务器的邀请码，请在下面输入。 silencedWarning: 显示这个页面是因为这些用户来自您的管理员设置的禁言服务器，所以他们有可能是垃圾信息。 isBot: 这个账号是一个机器人 @@ -1931,12 +1927,9 @@ moveTo: 将当前账号迁移至新账号 moveToLabel: 您要迁移到的目标账号： moveAccount: 迁移账号！ migrationConfirm: "您确实确定要将账号迁移到 {account} 吗？此操作无法撤消，并且您将无法再次正常使用旧账号。\n另外，请确保您已将此当前账号设置为要移出的账号。" -indexFromDescription: 留空以索引每个帖子 noteId: 帖子 ID moveFrom: 从旧账号迁移至此账号 defaultReaction: 发出和收到帖子的默认表情符号反应 -indexNotice: 现在开始索引。这可能需要一段时间，请至少一个小时内不要重新启动服务器。 -indexFrom: 从帖子 ID 开始的索引 sendModMail: 发送审核通知 isLocked: 该账号设置了关注请求 _filters: diff --git a/locales/zh-TW.yml b/locales/zh-TW.yml index ae89e38ae8..da573fd82b 100644 --- a/locales/zh-TW.yml +++ b/locales/zh-TW.yml @@ -1349,9 +1349,6 @@ _widgets: userList: 使用者列表 _userList: chooseList: 選擇一個清單 - meiliIndexCount: 編入索引的帖子 - meiliStatus: 伺服器狀態 - meiliSize: 索引大小 _cw: hide: "隱藏" show: "瀏覽更多" @@ -1879,8 +1876,6 @@ pushNotificationNotSupported: 你的瀏覽器或伺服器不支援推送通知 accessibility: 輔助功能 userSaysSomethingReasonReply: '{name} 回覆了包含 {reason} 的貼文' hiddenTags: 隱藏主題標籤 -indexPosts: 索引貼文 -indexNotice: 現在開始索引。 這可能需要一段時間，請不要在一個小時內重啟你的伺服器。 deleted: 已刪除 editNote: 編輯貼文 edited: '於 {date} {time} 編輯' @@ -1925,10 +1920,8 @@ sendModMail: 發送審核通知 enableIdenticonGeneration: 啟用Identicon生成 enableServerMachineStats: 啟用伺服器硬體統計資訊 reactionPickerSkinTone: 首選表情符號膚色 -indexFromDescription: 留空以索引每個貼文 preventAiLearning: 防止 AI 機器人抓取 preventAiLearningDescription: 請求第三方 AI 語言模型不要研究您上傳的內容，例如貼文和圖像。 -indexFrom: 建立此貼文ID以後的索引 isLocked: 該帳戶已獲得以下批准 isModerator: 板主 isAdmin: 管理員 diff --git a/packages/backend/migration/1698420787202-pgroonga.js b/packages/backend/migration/1698420787202-pgroonga.js new file mode 100644 index 0000000000..99638b4036 --- /dev/null +++ b/packages/backend/migration/1698420787202-pgroonga.js @@ -0,0 +1,21 @@ +export class Pgroonga1698420787202 { + name = "Pgroonga1698420787202"; + + async up(queryRunner) { + await queryRunner.query( + `CREATE INDEX "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `, + ); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_fcb770976ff8240af5799e3ffc"`); + await queryRunner.query(`DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"`); + await queryRunner.query(`DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index d5786f16ec..ecb9f80da3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,7 +25,6 @@ "@bull-board/koa": "5.14.2", "@bull-board/ui": "5.14.2", "@discordapp/twemoji": "^15.0.2", - "@elastic/elasticsearch": "8.12.2", "@koa/cors": "5.0.0", "@koa/multer": "3.0.2", "@koa/router": "12.0.1", @@ -81,7 +80,6 @@ "koa-send": "5.0.1", "koa-slow": "2.1.0", "megalodon": "workspace:*", - "meilisearch": "0.37.0", "mfm-js": "0.24.0", "mime-types": "2.1.35", "msgpackr": "^1.10.1", @@ -112,7 +110,6 @@ "sanitize-html": "2.12.1", "semver": "7.6.0", "sharp": "0.33.2", - "sonic-channel": "^1.3.1", "stringz": "2.1.0", "summaly": "2.7.0", "syslog-pro": "1.0.0", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 0389bf83fb..882e429c79 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -37,27 +37,6 @@ export type Source = { user?: string; tls?: { [z: string]: string }; }; - elasticsearch: { - host: string; - port: number; - ssl?: boolean; - user?: string; - pass?: string; - index?: string; - }; - sonic: { - host: string; - port: number; - auth?: string; - collection?: string; - bucket?: string; - }; - meilisearch: { - host: string; - port: number; - apiKey?: string; - ssl: boolean; - }; proxy?: string; proxySmtp?: string; diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index dc7493381c..58a9b1491b 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -2,7 +2,6 @@ import si from "systeminformation"; import Xev from "xev"; import * as osUtils from "os-utils"; import { fetchMeta } from "@/misc/fetch-meta.js"; -import meilisearch from "@/db/meilisearch.js"; const ev = new Xev(); @@ -30,7 +29,6 @@ export default function () { const memStats = await mem(); const netStats = await net(); const fsStats = await fs(); - const meilisearchStats = await meilisearchStatus(); const stats = { cpu: roundCpu(cpu), @@ -47,7 +45,6 @@ export default function () { r: round(Math.max(0, fsStats.rIO_sec ?? 0)), w: round(Math.max(0, fsStats.wIO_sec ?? 0)), }, - meilisearch: meilisearchStats, }; ev.emit("serverStats", stats); log.unshift(stats); @@ -86,16 +83,3 @@ async function fs() { const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 })); return data || { rIO_sec: 0, wIO_sec: 0 }; } - -// MEILI STAT -async function meilisearchStatus() { - if (meilisearch) { - return meilisearch.serverStats(); - } else { - return { - health: "unconfigured", - size: 0, - indexed_count: 0, - }; - } -} diff --git a/packages/backend/src/db/elasticsearch.ts b/packages/backend/src/db/elasticsearch.ts deleted file mode 100644 index 2640e7f918..0000000000 --- a/packages/backend/src/db/elasticsearch.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as elasticsearch from "@elastic/elasticsearch"; -import config from "@/config/index.js"; - -const index = { - settings: { - analysis: { - analyzer: { - ngram: { - tokenizer: "ngram", - }, - }, - }, - }, - mappings: { - properties: { - text: { - type: "text", - index: true, - analyzer: "ngram", - }, - userId: { - type: "keyword", - index: true, - }, - userHost: { - type: "keyword", - index: true, - }, - }, - }, -}; - -// Init ElasticSearch connection -const client = config.elasticsearch - ? new elasticsearch.Client({ - node: `${config.elasticsearch.ssl ? "https://" : "http://"}${ - config.elasticsearch.host - }:${config.elasticsearch.port}`, - auth: - config.elasticsearch.user && config.elasticsearch.pass - ? { - username: config.elasticsearch.user, - password: config.elasticsearch.pass, - } - : undefined, - pingTimeout: 30000, - }) - : null; - -if (client) { - client.indices - .exists({ - index: config.elasticsearch.index || "misskey_note", - }) - .then((exist) => { - if (!exist.body) { - client.indices.create({ - index: config.elasticsearch.index || "misskey_note", - body: index, - }); - } - }); -} - -export default client; diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts deleted file mode 100644 index 1f294058e5..0000000000 --- a/packages/backend/src/db/meilisearch.ts +++ /dev/null @@ -1,451 +0,0 @@ -import { Health, Index, MeiliSearch, Stats } from "meilisearch"; -import { dbLogger } from "./logger.js"; - -import config from "@/config/index.js"; -import { Note } from "@/models/entities/note.js"; -import * as url from "url"; -import { ILocalUser } from "@/models/entities/user.js"; -import { Followings, Users } from "@/models/index.js"; - -const logger = dbLogger.createSubLogger("meilisearch", "gray", false); - -let posts: Index; -let client: MeiliSearch; - -const hasConfig = - config.meilisearch && - (config.meilisearch.host || - config.meilisearch.port || - config.meilisearch.apiKey); - -if (hasConfig) { - const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; - const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; - const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; - const ssl = hasConfig ? config.meilisearch.ssl ?? false : false; - - logger.info("Connecting to MeiliSearch"); - - client = new MeiliSearch({ - host: `${ssl ? "https" : "http"}://${host}:${port}`, - apiKey: auth, - }); - - posts = client.index("posts"); - - posts - .updateSearchableAttributes(["text"]) - .catch((e) => - logger.error(`Setting searchable attr failed, searches won't work: ${e}`), - ); - - posts - .updateFilterableAttributes([ - "userName", - "userHost", - "mediaAttachment", - "createdAt", - "userId", - ]) - .catch((e) => - logger.error( - `Setting filterable attr failed, advanced searches won't work: ${e}`, - ), - ); - - posts - .updateSortableAttributes(["createdAt"]) - .catch((e) => - logger.error( - `Setting sortable attr failed, placeholder searches won't sort properly: ${e}`, - ), - ); - - posts - .updateStopWords([ - "the", - "a", - "as", - "be", - "of", - "they", - "these", - "is", - "are", - "これ", - "それ", - "あれ", - "この", - "その", - "あの", - "ここ", - "そこ", - "あそこ", - "こちら", - "どこ", - "私", - "僕", - "俺", - "君", - "あなた", - "我々", - "私達", - "彼女", - "彼", - "です", - "ます", - "は", - "が", - "の", - "に", - "を", - "で", - "へ", - "から", - "まで", - "より", - "も", - "どの", - "と", - "それで", - "しかし", - ]) - .catch((e) => - logger.error( - `Failed to set Meilisearch stop words, database size will be larger: ${e}`, - ), - ); - - posts - .updateRankingRules([ - "sort", - "words", - "typo", - "proximity", - "attribute", - "exactness", - ]) - .catch((e) => { - logger.error("Failed to set ranking rules, sorting won't work properly."); - }); - - logger.info("Connected to MeiliSearch"); -} - -export type MeilisearchNote = { - id: string; - text: string; - userId: string; - userHost: string; - userName: string; - channelId: string; - mediaAttachment: string; - createdAt: number; -}; - -function timestampToUnix(timestamp: string) { - let unix = 0; - - // Only contains numbers => UNIX timestamp - if (/^\d+$/.test(timestamp)) { - unix = Number.parseInt(timestamp); - } - - if (unix === 0) { - // Try to parse the timestamp as JavaScript Date - const date = Date.parse(timestamp); - if (Number.isNaN(date)) return 0; - unix = date / 1000; - } - - return unix; -} - -export default hasConfig - ? { - search: async ( - query: string, - limit: number, - offset: number, - userCtx: ILocalUser | null, - overrideSort: string | null, - ) => { - /// Advanced search syntax - /// from:user => filter by user + optional domain - /// has:image/video/audio/text/file => filter by attachment types - /// domain:domain.com => filter by domain - /// before:Date => show posts made before Date - /// after: Date => show posts made after Date - /// "text" => get posts with exact text between quotes - /// filter:following => show results only from users you follow - /// filter:followers => show results only from followers - /// order:desc/asc => order results ascending or descending - - const constructedFilters: string[] = []; - let sortRules: string[] = []; - - const splitSearch = query.split(" "); - - // Detect search operators and remove them from the actual query - const filteredSearchTerms = ( - await Promise.all( - splitSearch.map(async (term) => { - if (term.startsWith("has:")) { - const fileType = term.slice(4); - constructedFilters.push(`mediaAttachment = "${fileType}"`); - return null; - } else if (term.startsWith("from:")) { - let user = term.slice(5); - - if (user.length === 0) return null; - - // Cut off leading @, those aren't saved in the DB - if (user.charAt(0) === "@") { - user = user.slice(1); - } - - // Determine if we got a webfinger address or a single username - if (user.split("@").length > 1) { - const splitUser = user.split("@"); - - const domain = splitUser.pop(); - user = splitUser.join("@"); - - constructedFilters.push( - `userName = ${user} AND userHost = ${domain}`, - ); - } else { - constructedFilters.push(`userName = ${user}`); - } - - return null; - } else if (term.startsWith("domain:")) { - const domain = term.slice(7); - if ( - domain.length === 0 || - domain === "local" || - domain === config.hostname - ) { - constructedFilters.push("userHost NOT EXISTS"); - return null; - } - constructedFilters.push(`userHost = ${domain}`); - return null; - } else if (term.startsWith("after:")) { - const timestamp = term.slice(6); - - const unix = timestampToUnix(timestamp); - - if (unix !== 0) constructedFilters.push(`createdAt > ${unix}`); - - return null; - } else if (term.startsWith("before:")) { - const timestamp = term.slice(7); - - const unix = timestampToUnix(timestamp); - if (unix !== 0) constructedFilters.push(`createdAt < ${unix}`); - - return null; - } else if (term.startsWith("filter:following")) { - // Check if we got a context user - if (userCtx) { - // Fetch user follows from DB - const followedUsers = await Followings.find({ - where: { - followerId: userCtx.id, - }, - select: { - followeeId: true, - }, - }); - const followIDs = followedUsers.map( - (user) => user.followeeId, - ); - - if (followIDs.length === 0) return null; - - constructedFilters.push(`userId IN [${followIDs.join(",")}]`); - } else { - logger.warn( - "search filtered to follows called without user context", - ); - } - - return null; - } else if (term.startsWith("filter:followers")) { - // Check if we got a context user - if (userCtx) { - // Fetch users follows from DB - const followedUsers = await Followings.find({ - where: { - followeeId: userCtx.id, - }, - select: { - followerId: true, - }, - }); - const followIDs = followedUsers.map( - (user) => user.followerId, - ); - - if (followIDs.length === 0) return null; - - constructedFilters.push(`userId IN [${followIDs.join(",")}]`); - } else { - logger.warn( - "search filtered to followers called without user context", - ); - } - - return null; - } else if (term.startsWith("order:desc")) { - sortRules.push("createdAt:desc"); - - return null; - } else if (term.startsWith("order:asc")) { - sortRules.push("createdAt:asc"); - - return null; - } - - return term; - }), - ) - ).filter((term) => term !== null); - - // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search - // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want - // If the user has defined a sort rule, don't mess with it - if ( - filteredSearchTerms.length === 0 && - constructedFilters.length > 0 && - sortRules.length === 0 - ) { - sortRules.push("createdAt:desc"); - } - - // More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen. - if (sortRules.length > 1) { - sortRules = [sortRules[0]]; - } - - // An override sort takes precedence, user sorting is ignored here - if (overrideSort) { - sortRules = [overrideSort]; - } - - logger.info(`Searching for ${filteredSearchTerms.join(" ")}`); - logger.info(`Limit: ${limit}`); - logger.info(`Offset: ${offset}`); - logger.info(`Filters: ${constructedFilters}`); - logger.info(`Ordering: ${sortRules}`); - - return posts.search(filteredSearchTerms.join(" "), { - limit: limit, - offset: offset, - filter: constructedFilters, - sort: sortRules, - }); - }, - ingestNote: async (ingestNotes: Note | Note[]) => { - if (ingestNotes instanceof Note) { - ingestNotes = [ingestNotes]; - } - - const indexingBatch: MeilisearchNote[] = []; - - for (const note of ingestNotes) { - if (note.user === undefined) { - note.user = await Users.findOne({ - where: { - id: note.userId, - }, - }); - } - - let attachmentType = ""; - if (note.attachedFileTypes.length > 0) { - attachmentType = note.attachedFileTypes[0].split("/")[0]; - switch (attachmentType) { - case "image": - case "video": - case "audio": - case "text": - break; - default: - attachmentType = "file"; - break; - } - } - - indexingBatch.push({ - id: note.id.toString(), - text: note.text ? note.text : "", - userId: note.userId, - userHost: - note.userHost !== "" - ? note.userHost - : url.parse(config.host).host, - channelId: note.channelId ? note.channelId : "", - mediaAttachment: attachmentType, - userName: note.user?.username ?? "UNKNOWN", - createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy - }); - } - - return posts - .addDocuments(indexingBatch, { - primaryKey: "id", - }) - .then(() => - logger.info(`sent ${indexingBatch.length} posts for indexing`), - ); - }, - serverStats: async () => { - const health: Health = await client.health(); - const stats: Stats = await client.getStats(); - - return { - health: health.status, - size: stats.databaseSize, - indexed_count: stats.indexes["posts"].numberOfDocuments, - }; - }, - deleteNotes: async (note: Note | Note[] | string | string[]) => { - if (note instanceof Note) { - note = [note]; - } - if (typeof note === "string") { - note = [note]; - } - - const deletionBatch = note - .map((n) => { - if (n instanceof Note) { - return n.id; - } - - if (n.length > 0) return n; - - logger.error( - `Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify( - n, - )}`, - ); - - throw new Error( - `Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify( - n, - )}`, - ); - }) - .filter((el) => el !== null); - - await posts.deleteDocuments(deletionBatch as string[]).then(() => { - logger.info( - `submitted ${deletionBatch.length} large batch for deletion`, - ); - }); - }, - } - : null; diff --git a/packages/backend/src/db/sonic.ts b/packages/backend/src/db/sonic.ts deleted file mode 100644 index 032982f083..0000000000 --- a/packages/backend/src/db/sonic.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as SonicChannel from "sonic-channel"; -import { dbLogger } from "./logger.js"; - -import config from "@/config/index.js"; - -const logger = dbLogger.createSubLogger("sonic", "gray", false); - -const handlers = (type: string): SonicChannel.Handlers => ({ - connected: () => { - logger.succ(`Connected to Sonic ${type}`); - }, - disconnected: (error) => { - logger.warn(`Disconnected from Sonic ${type}, error: ${error}`); - }, - error: (error) => { - logger.warn(`Sonic ${type} error: ${error}`); - }, - retrying: () => { - logger.info(`Sonic ${type} retrying`); - }, - timeout: () => { - logger.warn(`Sonic ${type} timeout`); - }, -}); - -const hasConfig = - config.sonic && (config.sonic.host || config.sonic.port || config.sonic.auth); - -if (hasConfig) { - logger.info("Connecting to Sonic"); -} - -const host = hasConfig ? config.sonic.host ?? "localhost" : ""; -const port = hasConfig ? config.sonic.port ?? 1491 : 0; -const auth = hasConfig ? config.sonic.auth ?? "SecretPassword" : ""; -const collection = hasConfig ? config.sonic.collection ?? "main" : ""; -const bucket = hasConfig ? config.sonic.bucket ?? "default" : ""; - -export default hasConfig - ? { - search: new SonicChannel.Search({ host, port, auth }).connect( - handlers("search"), - ), - ingest: new SonicChannel.Ingest({ host, port, auth }).connect( - handlers("ingest"), - ), - - collection, - bucket, - } - : null; diff --git a/packages/backend/src/models/entities/drive-file.ts b/packages/backend/src/models/entities/drive-file.ts index 54f5a8ca28..5b31a7e336 100644 --- a/packages/backend/src/models/entities/drive-file.ts +++ b/packages/backend/src/models/entities/drive-file.ts @@ -70,6 +70,7 @@ export class DriveFile { }) public size: number; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: DB_MAX_IMAGE_COMMENT_LENGTH, nullable: true, diff --git a/packages/backend/src/models/entities/note.ts b/packages/backend/src/models/entities/note.ts index 8f7958e83d..e45fb02466 100644 --- a/packages/backend/src/models/entities/note.ts +++ b/packages/backend/src/models/entities/note.ts @@ -61,6 +61,7 @@ export class Note { }) public threadId: string | null; + @Index() // USING pgroonga @Column("text", { nullable: true, }) @@ -78,6 +79,7 @@ export class Note { }) public name: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 512, nullable: true, diff --git a/packages/backend/src/models/entities/user-profile.ts b/packages/backend/src/models/entities/user-profile.ts index 56887035ce..0cdf4e8393 100644 --- a/packages/backend/src/models/entities/user-profile.ts +++ b/packages/backend/src/models/entities/user-profile.ts @@ -38,6 +38,7 @@ export class UserProfile { }) public birthday: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 2048, nullable: true, diff --git a/packages/backend/src/models/entities/user.ts b/packages/backend/src/models/entities/user.ts index 07aba7badf..cb99fe53ca 100644 --- a/packages/backend/src/models/entities/user.ts +++ b/packages/backend/src/models/entities/user.ts @@ -58,6 +58,7 @@ export class User { }) public usernameLower: string; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 128, nullable: true, diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index ced08fee3b..58a0ae7486 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -13,7 +13,6 @@ import processDb from "./processors/db/index.js"; import processObjectStorage from "./processors/object-storage/index.js"; import processSystemQueue from "./processors/system/index.js"; import processWebhookDeliver from "./processors/webhook-deliver.js"; -import processBackground from "./processors/background/index.js"; import { endedPollNotification } from "./processors/ended-poll-notification.js"; import { queueLogger } from "./logger.js"; import { getJobInfo } from "./get-job-info.js"; @@ -482,14 +481,6 @@ export function createCleanRemoteFilesJob() { ); } -export function createIndexAllNotesJob(data = {}) { - return backgroundQueue.add("indexAllNotes", data, { - removeOnComplete: true, - removeOnFail: false, - timeout: 1000 * 60 * 60 * 24, - }); -} - export function webhookDeliver( webhook: Webhook, type: (typeof webhookEventTypes)[number], @@ -526,7 +517,6 @@ export default function () { webhookDeliverQueue.process(64, processWebhookDeliver); processDb(dbQueue); processObjectStorage(objectStorageQueue); - processBackground(backgroundQueue); systemQueue.add( "cleanCharts", diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts deleted file mode 100644 index 115f2147b6..0000000000 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type Bull from "bull"; -import type { DoneCallback } from "bull"; - -import { queueLogger } from "../../logger.js"; -import { Notes } from "@/models/index.js"; -import { MoreThan } from "typeorm"; -import { index } from "@/services/note/create.js"; -import { Note } from "@/models/entities/note.js"; -import meilisearch from "@/db/meilisearch.js"; -import { inspect } from "node:util"; - -const logger = queueLogger.createSubLogger("index-all-notes"); - -export default async function indexAllNotes( - job: Bull.Job>, - done: DoneCallback, -): Promise { - logger.info("Indexing all notes..."); - - let cursor: string | null = (job.data.cursor as string) ?? null; - let indexedCount: number = (job.data.indexedCount as number) ?? 0; - let total: number = (job.data.total as number) ?? 0; - - let running = true; - const take = 10000; - const batch = 100; - while (running) { - logger.info( - `Querying for ${take} notes ${indexedCount}/${ - total ? total : "?" - } at ${cursor}`, - ); - - let notes: Note[] = []; - try { - notes = await Notes.find({ - where: { - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: take, - order: { - id: 1, - }, - relations: ["user"], - }); - } catch (e: any) { - logger.error(`Failed to query notes:\n${inspect(e)}`); - done(e); - break; - } - - if (notes.length === 0) { - await job.progress(100); - running = false; - break; - } - - try { - const count = await Notes.count(); - total = count; - await job.update({ indexedCount, cursor, total }); - } catch (e) {} - - for (let i = 0; i < notes.length; i += batch) { - const chunk = notes.slice(i, i + batch); - - if (meilisearch) { - await meilisearch.ingestNote(chunk); - } - - await Promise.all(chunk.map((note) => index(note, true))); - - indexedCount += chunk.length; - const pct = (indexedCount / total) * 100; - await job.update({ indexedCount, cursor, total }); - await job.progress(+pct.toFixed(1)); - logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`); - } - cursor = notes[notes.length - 1].id; - await job.update({ indexedCount, cursor, total }); - - if (notes.length < take) { - running = false; - } - } - - done(); - logger.info("All notes have been indexed."); -} diff --git a/packages/backend/src/queue/processors/background/index.ts b/packages/backend/src/queue/processors/background/index.ts deleted file mode 100644 index 6674f954b0..0000000000 --- a/packages/backend/src/queue/processors/background/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type Bull from "bull"; -import indexAllNotes from "./index-all-notes.js"; - -const jobs = { - indexAllNotes, -} as Record>>; - -export default function (q: Bull.Queue) { - for (const [k, v] of Object.entries(jobs)) { - q.process(k, 16, v); - } -} diff --git a/packages/backend/src/queue/processors/db/delete-account.ts b/packages/backend/src/queue/processors/db/delete-account.ts index adb93484bd..b43cdd137c 100644 --- a/packages/backend/src/queue/processors/db/delete-account.ts +++ b/packages/backend/src/queue/processors/db/delete-account.ts @@ -7,7 +7,6 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import { MoreThan } from "typeorm"; import { deleteFileSync } from "@/services/drive/delete-file.js"; import { sendEmail } from "@/services/send-email.js"; -import meilisearch from "@/db/meilisearch.js"; const logger = queueLogger.createSubLogger("delete-account"); @@ -42,9 +41,6 @@ export async function deleteAccount( cursor = notes[notes.length - 1].id; await Notes.delete(notes.map((note) => note.id)); - if (meilisearch) { - await meilisearch.deleteNotes(notes); - } } logger.succ("All of notes deleted"); diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 8184df5d23..fa029c1f4a 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -51,7 +51,6 @@ import * as ep___admin_relays_list from "./endpoints/admin/relays/list.js"; import * as ep___admin_relays_remove from "./endpoints/admin/relays/remove.js"; import * as ep___admin_resetPassword from "./endpoints/admin/reset-password.js"; import * as ep___admin_resolveAbuseUserReport from "./endpoints/admin/resolve-abuse-user-report.js"; -import * as ep___admin_search_indexAll from "./endpoints/admin/search/index-all.js"; import * as ep___admin_sendEmail from "./endpoints/admin/send-email.js"; import * as ep___admin_sendModMail from "./endpoints/admin/send-mod-mail.js"; import * as ep___admin_serverInfo from "./endpoints/admin/server-info.js"; @@ -400,7 +399,6 @@ const eps = [ ["admin/relays/remove", ep___admin_relays_remove], ["admin/reset-password", ep___admin_resetPassword], ["admin/resolve-abuse-user-report", ep___admin_resolveAbuseUserReport], - ["admin/search/index-all", ep___admin_search_indexAll], ["admin/send-email", ep___admin_sendEmail], ["admin/send-mod-mail", ep___admin_sendModMail], ["admin/server-info", ep___admin_serverInfo], diff --git a/packages/backend/src/server/api/endpoints/admin/search/index-all.ts b/packages/backend/src/server/api/endpoints/admin/search/index-all.ts deleted file mode 100644 index 531428849b..0000000000 --- a/packages/backend/src/server/api/endpoints/admin/search/index-all.ts +++ /dev/null @@ -1,28 +0,0 @@ -import define from "@/server/api/define.js"; -import { createIndexAllNotesJob } from "@/queue/index.js"; - -export const meta = { - tags: ["admin"], - - requireCredential: true, - requireModerator: true, -} as const; - -export const paramDef = { - type: "object", - properties: { - cursor: { - type: "string", - format: "misskey:id", - nullable: true, - default: null, - }, - }, - required: [], -} as const; - -export default define(meta, paramDef, async (ps, _me) => { - createIndexAllNotesJob({ - cursor: ps.cursor ?? undefined, - }); -}); diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 8e818745fc..f8a7a626b7 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -1,5 +1,4 @@ import { In } from "typeorm"; -import { index } from "@/services/note/create.js"; import type { IRemoteUser, User } from "@/models/entities/user.js"; import { Users, @@ -626,8 +625,6 @@ export default define(meta, paramDef, async (ps, user) => { } if (publishing && user.isIndexable) { - index(note, true); - // Publish update event for the updated note details publishNoteStream(note.id, "updated", { updatedAt: update.updatedAt, diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 9374e0bac7..6846a7dd14 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -1,10 +1,5 @@ -import { FindManyOptions, In } from "typeorm"; import { Notes } from "@/models/index.js"; import { Note } from "@/models/entities/note.js"; -import config from "@/config/index.js"; -import es from "@/db/elasticsearch.js"; -import sonic from "@/db/sonic.js"; -import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js"; import define from "@/server/api/define.js"; import { makePaginationQuery } from "@/server/api/common/make-pagination-query.js"; import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js"; @@ -69,282 +64,43 @@ export const paramDef = { } as const; export default define(meta, paramDef, async (ps, me) => { - if (es == null && sonic == null && meilisearch == null) { - const query = makePaginationQuery( - Notes.createQueryBuilder("note"), - ps.sinceId, - ps.untilId, - ); + const query = makePaginationQuery( + Notes.createQueryBuilder("note"), + ps.sinceId, + ps.untilId, + ); - if (ps.userId != null) { - query.andWhere("note.userId = :userId", { userId: ps.userId }); - } - - if (ps.channelId != null) { - query.andWhere("note.channelId = :channelId", { - channelId: ps.channelId, - }); - } - - query - .andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` }) - .andWhere("note.visibility = 'public'") - .innerJoinAndSelect("note.user", "user") - .andWhere("user.isIndexable = TRUE") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); - - generateVisibilityQuery(query, me); - if (me) generateMutedUserQuery(query, me); - if (me) generateBlockedUserQuery(query, me); - - const notes: Note[] = await query.take(ps.limit).getMany(); - - return await Notes.packMany(notes, me); - } else if (sonic) { - let start = 0; - const chunkSize = 100; - - // Use sonic to fetch and step through all search results that could match the requirements - const ids = []; - while (true) { - const results = await sonic.search.query( - sonic.collection, - sonic.bucket, - ps.query, - { - limit: chunkSize, - offset: start, - }, - ); - - start += chunkSize; - - if (results.length === 0) { - break; - } - - const res = results - .map((k) => JSON.parse(k)) - .filter((key) => { - if (ps.userId && key.userId !== ps.userId) { - return false; - } - if (ps.channelId && key.channelId !== ps.channelId) { - return false; - } - if (ps.sinceId && key.id <= ps.sinceId) { - return false; - } - if (ps.untilId && key.id >= ps.untilId) { - return false; - } - return true; - }) - .map((key) => key.id); - - ids.push(...res); - } - - // Sort all the results by note id DESC (newest first) - ids.sort((a, b) => b - a); - - // Fetch the notes from the database until we have enough to satisfy the limit - start = 0; - const found = []; - while (found.length < ps.limit && start < ids.length) { - const chunk = ids.slice(start, start + chunkSize); - const notes: Note[] = await Notes.find({ - where: { - id: In(chunk), - }, - }); - - // The notes are checked for visibility and muted/blocked users when packed - found.push(...(await Notes.packMany(notes, me))); - start += chunkSize; - } - - // If we have more results than the limit, trim them - if (found.length > ps.limit) { - found.length = ps.limit; - } - - return found; - } else if (meilisearch) { - let start = 0; - const chunkSize = 100; - const sortByDate = ps.order !== "relevancy"; - - type NoteResult = { - id: string; - createdAt: number; - }; - const extractedNotes: NoteResult[] = []; - - while (true) { - const searchRes = await meilisearch.search( - ps.query, - chunkSize, - start, - me, - sortByDate ? "createdAt:desc" : null, - ); - const results: MeilisearchNote[] = searchRes.hits as MeilisearchNote[]; - - start += chunkSize; - - if (results.length === 0) { - break; - } - - const res = results - .filter((key: MeilisearchNote) => { - if (ps.userId && key.userId !== ps.userId) { - return false; - } - if (ps.channelId && key.channelId !== ps.channelId) { - return false; - } - if (ps.sinceId && key.id <= ps.sinceId) { - return false; - } - if (ps.untilId && key.id >= ps.untilId) { - return false; - } - return true; - }) - .map((key) => { - return { - id: key.id, - createdAt: key.createdAt, - }; - }); - - extractedNotes.push(...res); - } - - // Fetch the notes from the database until we have enough to satisfy the limit - start = 0; - const found = []; - const noteIDs = extractedNotes.map((note) => note.id); - - // Index the ID => index number into a map, so we can restore the array ordering efficiently later - const idIndexMap = new Map(noteIDs.map((id, index) => [id, index])); - - while (found.length < ps.limit && start < noteIDs.length) { - const chunk = noteIDs.slice(start, start + chunkSize); - - let query: FindManyOptions = { - where: { - id: In(chunk), - }, - }; - - const notes: Note[] = await Notes.find(query); - - // Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier) - // @ts-ignore - notes.sort((a, b) => idIndexMap.get(a.id) - idIndexMap.get(b.id)); - - // The notes are checked for visibility and muted/blocked users when packed - found.push(...(await Notes.packMany(notes, me))); - start += chunkSize; - } - - // If we have more results than the limit, trim the results down - if (found.length > ps.limit) { - found.length = ps.limit; - } - - return found; - } else { - const userQuery = - ps.userId != null - ? [ - { - term: { - userId: ps.userId, - }, - }, - ] - : []; - - const hostQuery = - ps.userId == null - ? ps.host === null - ? [ - { - bool: { - must_not: { - exists: { - field: "userHost", - }, - }, - }, - }, - ] - : ps.host !== undefined - ? [ - { - term: { - userHost: ps.host, - }, - }, - ] - : [] - : []; - - const result = await es.search({ - index: config.elasticsearch.index || "misskey_note", - body: { - size: ps.limit, - from: ps.offset, - query: { - bool: { - must: [ - { - simple_query_string: { - fields: ["text"], - query: ps.query.toLowerCase(), - default_operator: "and", - }, - }, - ...hostQuery, - ...userQuery, - ], - }, - }, - sort: [ - { - _doc: "desc", - }, - ], - }, - }); - - const hits = result.body.hits.hits.map((hit: any) => hit._id); - - if (hits.length === 0) return []; - - // Fetch found notes - const notes = await Notes.find({ - where: { - id: In(hits), - }, - order: { - id: -1, - }, - }); - - return await Notes.packMany(notes, me); + if (ps.userId != null) { + query.andWhere("note.userId = :userId", { userId: ps.userId }); } + + if (ps.channelId != null) { + query.andWhere("note.channelId = :channelId", { + channelId: ps.channelId, + }); + } + + query + .andWhere("note.text &@~ :q", { q: `${sqlLikeEscape(ps.query)}` }) + .andWhere("note.visibility = 'public'") + .innerJoinAndSelect("note.user", "user") + .andWhere("user.isIndexable = TRUE") + .leftJoinAndSelect("user.avatar", "avatar") + .leftJoinAndSelect("user.banner", "banner") + .leftJoinAndSelect("note.reply", "reply") + .leftJoinAndSelect("note.renote", "renote") + .leftJoinAndSelect("reply.user", "replyUser") + .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") + .leftJoinAndSelect("replyUser.banner", "replyUserBanner") + .leftJoinAndSelect("renote.user", "renoteUser") + .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") + .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + + generateVisibilityQuery(query, me); + if (me) generateMutedUserQuery(query, me); + if (me) generateBlockedUserQuery(query, me); + + const notes: Note[] = await query.take(ps.limit).getMany(); + + return await Notes.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index ff3bfcd100..d3b6a08074 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -1,7 +1,6 @@ import * as os from "node:os"; import si from "systeminformation"; import define from "@/server/api/define.js"; -import meilisearch from "@/db/meilisearch.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; export const meta = { @@ -63,15 +62,3 @@ export default define(meta, paramDef, async () => { }, }; }); - -async function meilisearchStatus() { - if (meilisearch) { - return meilisearch.serverStats(); - } else { - return { - health: "unconfigured", - size: 0, - indexed_count: 0, - }; - } -} diff --git a/packages/backend/src/server/api/endpoints/users/search.ts b/packages/backend/src/server/api/endpoints/users/search.ts index 3aef6dc901..a15a0feb4b 100644 --- a/packages/backend/src/server/api/endpoints/users/search.ts +++ b/packages/backend/src/server/api/endpoints/users/search.ts @@ -78,8 +78,8 @@ export default define(meta, paramDef, async (ps, me) => { const nameQuery = Users.createQueryBuilder("user") .where( new Brackets((qb) => { - qb.where("user.name ILIKE :query", { - query: `%${sqlLikeEscape(ps.query)}%`, + qb.where("user.name &@~ :query", { + query: `${sqlLikeEscape(ps.query)}`, }); // Also search username if it qualifies as username @@ -115,8 +115,8 @@ export default define(meta, paramDef, async (ps, me) => { if (users.length < ps.limit) { const profQuery = UserProfiles.createQueryBuilder("prof") .select("prof.userId") - .where("prof.description ILIKE :query", { - query: `%${sqlLikeEscape(ps.query)}%`, + .where("prof.description &@~ :query", { + query: `${sqlLikeEscape(ps.query)}`, }); if (ps.origin === "local") { diff --git a/packages/backend/src/server/nodeinfo.ts b/packages/backend/src/server/nodeinfo.ts index f15eb8c6e2..73266f26a7 100644 --- a/packages/backend/src/server/nodeinfo.ts +++ b/packages/backend/src/server/nodeinfo.ts @@ -82,7 +82,6 @@ const nodeinfo2 = async () => { disableRecommendedTimeline: meta.disableRecommendedTimeline, disableGlobalTimeline: meta.disableGlobalTimeline, emailRequiredForSignup: meta.emailRequiredForSignup, - searchFilters: config.meilisearch ? true : false, postEditing: true, postImports: meta.experimentalFeatures?.postImports || false, enableHcaptcha: meta.enableHcaptcha, diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 59665e2219..d2ec137d38 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -1,6 +1,4 @@ import * as mfm from "mfm-js"; -import es from "@/db/elasticsearch.js"; -import sonic from "@/db/sonic.js"; import { publishMainStream, publishNotesStream, @@ -59,7 +57,6 @@ import type { UserProfile } from "@/models/entities/user-profile.js"; import { db } from "@/db/postgre.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; -import meilisearch from "@/db/meilisearch.js"; import { redisClient } from "@/db/redis.js"; import { Mutex } from "redis-semaphore"; import { langmap } from "@/misc/langmap.js"; @@ -166,7 +163,6 @@ export default async ( createdAt: User["createdAt"]; isBot: User["isBot"]; inbox?: User["inbox"]; - isIndexable?: User["isIndexable"]; }, data: Option, silent = false, @@ -654,11 +650,6 @@ export default async ( } }); } - - // Register to search database - if (user.isIndexable) { - await index(note, false); - } }); async function renderNoteOrRenoteActivity(data: Option, note: Note) { @@ -814,40 +805,6 @@ async function insertNote( } } -export async function index(note: Note, reindexing: boolean): Promise { - if (!note.text || note.visibility !== "public") return; - - if (config.elasticsearch && es) { - es.index({ - index: config.elasticsearch.index || "misskey_note", - id: note.id.toString(), - body: { - text: normalizeForSearch(note.text), - userId: note.userId, - userHost: note.userHost, - }, - }); - } - - if (sonic) { - await sonic.ingest.push( - sonic.collection, - sonic.bucket, - JSON.stringify({ - id: note.id, - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - }), - note.text, - ); - } - - if (meilisearch && !reindexing) { - await meilisearch.ingestNote(note); - } -} - async function notifyToWatchersOfRenotee( renote: Note, user: { id: User["id"] }, diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index f22fc35eba..988e1c8c48 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -16,7 +16,6 @@ import { import { countSameRenotes } from "@/misc/count-same-renotes.js"; import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js"; import { deliverToRelays } from "@/services/relay.js"; -import meilisearch from "@/db/meilisearch.js"; /** * 投稿を削除します。 @@ -117,10 +116,6 @@ export default async function ( userId: user.id, }); } - - if (meilisearch) { - await meilisearch.deleteNotes(note.id); - } } async function findCascadingNotes(note: Note) { diff --git a/packages/client/src/pages/admin/database.vue b/packages/client/src/pages/admin/database.vue index e2da648123..c746c7032e 100644 --- a/packages/client/src/pages/admin/database.vue +++ b/packages/client/src/pages/admin/database.vue @@ -7,9 +7,6 @@ :display-back-button="true" /> - {{ - i18n.ts.indexPosts - }} diff --git a/packages/client/src/pages/admin/index.vue b/packages/client/src/pages/admin/index.vue index ee5bbabb33..1dd8540414 100644 --- a/packages/client/src/pages/admin/index.vue +++ b/packages/client/src/pages/admin/index.vue @@ -78,7 +78,6 @@ import * as os from "@/os"; import { lookupUser } from "@/scripts/lookup-user"; import { lookupFile } from "@/scripts/lookup-file"; import { lookupInstance } from "@/scripts/lookup-instance"; -import { indexPosts } from "@/scripts/index-posts"; import { defaultStore } from "@/store"; import { useRouter } from "@/router"; import { @@ -156,16 +155,6 @@ const menuDef = computed(() => [ }, ] : []), - ...($i.isAdmin - ? [ - { - type: "button", - icon: `${icon("ph-list-magnifying-glass")}`, - text: i18n.ts.indexPosts, - action: indexPosts, - }, - ] - : []), ], }, { diff --git a/packages/client/src/pages/admin/overview.metrics.vue b/packages/client/src/pages/admin/overview.metrics.vue index c8d7efadfb..498659b194 100644 --- a/packages/client/src/pages/admin/overview.metrics.vue +++ b/packages/client/src/pages/admin/overview.metrics.vue @@ -29,24 +29,6 @@

Used: {{ bytes(diskUsed, 1) }}

- -
- -
-

MeiliSearch

-

- {{ i18n.ts._widgets.meiliStatus }}: {{ meiliAvailable }} -

-

- {{ i18n.ts._widgets.meiliSize }}: - {{ bytes(meiliTotalSize, 1) }} -

-

- {{ i18n.ts._widgets.meiliIndexCount }}: - {{ meiliIndexCount }} -

-
-
@@ -57,7 +39,6 @@ import XPie from "../../widgets/server-metric/pie.vue"; import bytes from "@/filters/bytes"; import { useStream } from "@/stream"; import * as os from "@/os"; -import { i18n } from "@/i18n"; import icon from "@/scripts/icon"; const stream = useStream(); @@ -72,11 +53,6 @@ const memTotal = ref(0); const memUsed = ref(0); const memFree = ref(0); -const meiliProgress = ref(0); -const meiliTotalSize = ref(0); -const meiliIndexCount = ref(0); -const meiliAvailable = ref("unavailable"); - const diskUsage = computed(() => meta.fs.used / meta.fs.total); const diskTotal = computed(() => meta.fs.total); const diskUsed = computed(() => meta.fs.used); @@ -89,11 +65,6 @@ function onStats(stats) { memTotal.value = stats.mem.total; memUsed.value = stats.mem.active; memFree.value = memTotal.value - memUsed.value; - - meiliTotalSize.value = stats.meilisearch.size; - meiliIndexCount.value = stats.meilisearch.indexed_count; - meiliAvailable.value = stats.meilisearch.health; - meiliProgress.value = meiliIndexCount.value / serverStats.notesCount; } const connection = stream.useChannel("serverStats"); diff --git a/packages/client/src/scripts/index-posts.ts b/packages/client/src/scripts/index-posts.ts deleted file mode 100644 index 94b545e90e..0000000000 --- a/packages/client/src/scripts/index-posts.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { i18n } from "@/i18n"; -import * as os from "@/os"; - -export async function indexPosts() { - const { canceled, result: index } = await os.inputText({ - title: i18n.ts.indexFrom, - text: i18n.ts.indexFromDescription, - }); - if (canceled) return; - - if (index == null || index === "") { - await os.api("admin/search/index-all"); - await os.alert({ - type: "info", - text: i18n.ts.indexNotice, - }); - } else { - await os.api("admin/search/index-all", { - cursor: index, - }); - await os.alert({ - type: "info", - text: i18n.ts.indexNotice, - }); - } -} diff --git a/packages/client/src/widgets/server-metric/index.vue b/packages/client/src/widgets/server-metric/index.vue index 9276f5a110..a9026efeac 100644 --- a/packages/client/src/widgets/server-metric/index.vue +++ b/packages/client/src/widgets/server-metric/index.vue @@ -46,13 +46,6 @@ :connection="connection" :meta="meta" /> - @@ -66,7 +59,6 @@ import XNet from "./net.vue"; import XCpu from "./cpu.vue"; import XMemory from "./mem.vue"; import XDisk from "./disk.vue"; -import XMeili from "./meilisearch.vue"; import MkContainer from "@/components/MkContainer.vue"; import type { GetFormResultType } from "@/scripts/form"; import * as os from "@/os"; diff --git a/packages/client/src/widgets/server-metric/meilisearch.vue b/packages/client/src/widgets/server-metric/meilisearch.vue deleted file mode 100644 index ea9d1120a7..0000000000 --- a/packages/client/src/widgets/server-metric/meilisearch.vue +++ /dev/null @@ -1,86 +0,0 @@ - - - - - diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ad105accc..3715d96ad7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,9 +75,6 @@ importers: '@discordapp/twemoji': specifier: ^15.0.2 version: 15.0.2 - '@elastic/elasticsearch': - specifier: 8.12.2 - version: 8.12.2 '@koa/cors': specifier: 5.0.0 version: 5.0.0 @@ -243,9 +240,6 @@ importers: megalodon: specifier: workspace:* version: link:../megalodon - meilisearch: - specifier: 0.37.0 - version: 0.37.0 mfm-js: specifier: 0.24.0 version: 0.24.0 @@ -336,9 +330,6 @@ importers: sharp: specifier: 0.33.2 version: 0.33.2 - sonic-channel: - specifier: ^1.3.1 - version: 1.3.1 stringz: specifier: 2.1.0 version: 2.1.0 @@ -1889,30 +1880,6 @@ packages: universalify: 0.1.2 dev: false - /@elastic/elasticsearch@8.12.2: - resolution: {integrity: sha512-04NvH3LIgcv1Uwguorfw2WwzC9Lhfsqs9f0L6uq6MrCw0lqe/HOQ6E8vJ6EkHAA15iEfbhtxOtenbZVVcE+mAQ==} - engines: {node: '>=18'} - dependencies: - '@elastic/transport': 8.4.1 - tslib: 2.6.2 - transitivePeerDependencies: - - supports-color - dev: false - - /@elastic/transport@8.4.1: - resolution: {integrity: sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA==} - engines: {node: '>=16'} - dependencies: - debug: 4.3.4(supports-color@8.1.1) - hpagent: 1.2.0 - ms: 2.1.3 - secure-json-parse: 2.7.0 - tslib: 2.6.2 - undici: 5.23.0 - transitivePeerDependencies: - - supports-color - dev: false - /@emnapi/runtime@0.45.0: resolution: {integrity: sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==} requiresBuild: true @@ -7182,6 +7149,7 @@ packages: node-fetch: 2.6.12 transitivePeerDependencies: - encoding + dev: true /cross-spawn@5.1.0: resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} @@ -12529,14 +12497,6 @@ packages: engines: {node: '>= 0.6'} dev: false - /meilisearch@0.37.0: - resolution: {integrity: sha512-LdbK6JmRghCawrmWKJSEQF0OiE82md+YqJGE/U2JcCD8ROwlhTx0KM6NX4rQt0u0VpV0QZVG9umYiu3CSSIJAQ==} - dependencies: - cross-fetch: 3.1.8 - transitivePeerDependencies: - - encoding - dev: false - /memoize@10.0.0: resolution: {integrity: sha512-H6cBLgsi6vMWOcCpvVCdFFnl3kerEXbrYh9q+lY6VXvQSmM6CkmV08VOwT+WE2tzIEqRPFfAq3fm4v/UIW6mSA==} engines: {node: '>=18'} @@ -15181,10 +15141,6 @@ packages: ajv-keywords: 3.5.2(ajv@6.12.6) dev: true - /secure-json-parse@2.7.0: - resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} - dev: false - /seedrandom@2.4.2: resolution: {integrity: sha512-uQ72txMoObtuJooiBLSVs5Yu2e9d/lHQz0boaqHjW8runXB9vR8nFtaZV54wYii613N0C8ZqTBLsfwDhAdpvqQ==} dev: false @@ -15462,11 +15418,6 @@ packages: smart-buffer: 4.2.0 dev: false - /sonic-channel@1.3.1: - resolution: {integrity: sha512-+K4IZVFE7Tf2DB4EFZ23xo7a/+gJaiOHhFzXVZpzkX6Rs/rvf4YbSxnEGdYw8mrTcjtpG+jLVQEhP8sNTtN5VA==} - engines: {node: '>= 6.0.0'} - dev: false - /sort-keys-length@1.0.1: resolution: {integrity: sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw==} engines: {node: '>=0.10.0'}