refactor: Iceshrimp’s Mastodon API implementation with original extensions

Co-authored-by: Eana Hufwe <eana@1a23.com>
Co-authored-by: AkiraFukushima <h3.poteto@gmail.com>

.gitignore

@@ -55,9 +55,6 @@ packages/backend/assets/instance.css
 packages/backend/assets/sounds/None.mp3
 packages/backend/assets/LICENSE
 
-packages/megalodon/lib
-packages/megalodon/.idea
-
 dev/container/firefish
 dev/container/db
 dev/container/redis

.gitlab-ci.yml

@@ -106,7 +106,6 @@ test:build:backend_ts_only:
         paths:
           - packages/backend/**/*
           - packages/firefish-js/**/*
-          - packages/megalodon/**/*
       when: always
   before_script:
     - apt-get update && apt-get -y upgrade
@@ -123,7 +122,7 @@ test:build:backend_ts_only:
     - psql --host postgres --user "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" --command 'CREATE EXTENSION pgroonga'
   script:
     - pnpm install --frozen-lockfile
-    - pnpm --filter 'backend' --filter 'firefish-js' --filter 'megalodon' run build:debug
+    - pnpm --filter 'backend' --filter 'firefish-js' run build:debug
     - pnpm run migrate
     - psql --host postgres --user "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" --command "$(cat docs/downgrade.sql)"
 

Dockerfile

@@ -39,7 +39,6 @@ COPY packages/backend/package.json packages/backend/package.json
 COPY packages/client/package.json packages/client/package.json
 COPY packages/sw/package.json packages/sw/package.json
 COPY packages/firefish-js/package.json packages/firefish-js/package.json
-COPY packages/megalodon/package.json packages/megalodon/package.json
 COPY pnpm-lock.yaml ./
 
 # Install dev mode dependencies for compilation
@@ -63,8 +62,6 @@ RUN apk update && apk add --no-cache zip unzip tini ffmpeg curl
 
 COPY . ./
 
-COPY --from=build /firefish/packages/megalodon /firefish/packages/megalodon
-
 # Copy node modules
 COPY --from=build /firefish/node_modules /firefish/node_modules
 COPY --from=build /firefish/packages/backend/node_modules /firefish/packages/backend/node_modules

biome.json

@@ -26,6 +26,20 @@
 					}
 				}
 			}
+		},
+		{
+			"include": ["packages/backend/src/server/api/mastodon/**/*.ts"],
+			"linter": {
+				"rules": {
+					"style": {
+						"noParameterAssign": "off"
+					},
+					"complexity": {
+						"noStaticOnlyClass": "off",
+						"noThisInStatic": "off"
+					}
+				}
+			}
 		}
 	]
 }

docs/api-change.md

@@ -2,6 +2,12 @@
 
 Breaking changes are indicated by the :warning: icon.
 
+## Unreleased
+
+- The old Mastodon API has been replaced with a new implementation based on Iceshrimp’s. **Breaking changes:**
+  - The new API uses a new format to manage Mastodon sessions in the database, whereas old implementation uses Misskey sessions. All previous client app and token registrations will not work with the new API. All clients need to be re-registered and all users need to re-authenticate.
+  - All IDs (of statuses/notes, notifications, users, etc.) will be using the alphanumerical format, aligning with the Firefish/Misskey API. The old numerical IDs will not work when queried against the new API.
+
 ## v20240523
 
 - Added `scheduledAt` optional parameter to `notes/create` (!10789)

docs/changelog.md

@@ -5,6 +5,10 @@ Critical security updates are indicated by the :warning: icon.
 - Server administrators should check [notice-for-admins.md](./notice-for-admins.md) as well.
 - Third-party client/bot developers may want to check [api-change.md](./api-change.md) as well.
 
+## Unreleased
+
+- Ported Mastodon API support from Iceshrimp, with added Firefish extensions including push notifications, post languages, schedule post support, and more.
+
 ## [v20240523](https://firefish.dev/firefish/firefish/-/merge_requests/10898/commits)
 
 - Add scheduled posts

docs/downgrade.sql

@@ -30,7 +30,10 @@ DELETE FROM "migrations" WHERE name IN (
     'EmojiModerator1692825433698',
     'RemoveNsfwDetection1705848938166',
     'FirefishUrlMove1707850084123',
-    'RemoveNativeUtilsMigration1705877093218'
+    'SwSubscriptionAccessToken1709395223611'
+    'UserProfileMentions1711075007936',
+    'ClientCredentials1713108561474',
+    'AddMastodonSubscriptionType1715181461692',
 );
 
 -- remove-enum-typename-suffix
@@ -784,4 +787,16 @@ VALUES
     ('m20231002_143323_remove_integrations', 1705876632)
 ;
 
+-- sw-subscription-per-access-token
+ALTER TABLE "sw_subscription" DROP CONSTRAINT "FK_98a1aa2db2a5253924f42f38767";
+ALTER TABLE "sw_subscription" DROP COLUMN "appAccessTokenId";
+ALTER TABLE "sw_subscription" DROP COLUMN "subscriptionTypes";
+
+-- user-profile-mentions
+ALTER TABLE "user_profile" DROP COLUMN "mentions";
+
+-- client-credential-support
+ALTER TABLE "access_token" ALTER COLUMN "userId" SET NOT NULL;
+
+
 COMMIT;

docs/notice-for-admins.md

@@ -8,6 +8,12 @@ You can skip intermediate versions when upgrading from an old version, but pleas
 
 Required Node.js version has been bumped from v18.17.0 to v18.19.0. Also, as written in the [v20240430 note](https://firefish.dev/firefish/firefish/-/blob/d3394b97f021dea323ec3ae36e39930680242482/docs/notice-for-admins.md#v20240430), it is highly recommended that you use an even newer version since v18.19.0 has known vulnerabilities.
 
+### For Mastodon API users
+
+The old Mastodon API has been replaced with a new implementation based on Iceshrimp’s. **Breaking changes:**
+- The new API uses a new format to manage Mastodon sessions in the database, whereas old implementation uses Misskey sessions. All previous client app and token registrations will not work with the new API. All clients need to be re-registered and all users need to re-authenticate.
+- All IDs (of statuses/notes, notifications, users, etc.) will be using the alphanumerical format, aligning with the Firefish/Misskey API. The old numerical IDs will not work when queried against the new API.
+
 ## v20240523
 
 ### For all users

locales/en-US.yml

@@ -293,6 +293,7 @@ usernameOrUserId: "Username or user id"
 noSuchUser: "User not found"
 lookup: "Lookup"
 announcements: "Announcements"
+announcement: "Announcement"
 imageUrl: "Image URL"
 remove: "Delete"
 removed: "Successfully deleted"

locales/ja-JP.yml

@@ -254,6 +254,7 @@ usernameOrUserId: "ユーザー名かユーザーID"
 noSuchUser: "ユーザーが見つかりません"
 lookup: "照会"
 announcements: "お知らせ"
+announcement: "お知らせ"
 imageUrl: "画像URL"
 remove: "削除"
 removed: "削除しました"

locales/zh-CN.yml

@@ -243,6 +243,7 @@ usernameOrUserId: "用户名或用户 ID"
 noSuchUser: "用户不存在"
 lookup: "查询"
 announcements: "公告"
+announcement: "公告"
 imageUrl: "图片 URL"
 remove: "删除"
 removed: "已删除"

locales/zh-TW.yml

@@ -242,6 +242,7 @@ usernameOrUserId: "使用者名稱或使用者ID"
 noSuchUser: "使用者不存在"
 lookup: "查詢"
 announcements: "公告"
+announcement: "公告"
 imageUrl: "圖片URL"
 remove: "刪除"
 removed: "已成功刪除"

packages/README.md

@@ -8,4 +8,3 @@ This directory contains all of the packages Firefish uses.
 - `client`: Web interface written in Vue3 and TypeScript
 - `sw`: Web [Service Worker](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) written in TypeScript
 - `firefish-js`: TypeScript SDK for both backend and client
-- `megalodon`: TypeScript library used for partial Mastodon API compatibility

packages/backend-rs/src/model/entity/access_token.rs

@@ -17,7 +17,7 @@ pub struct Model {
     pub token: String,
     pub hash: String,
     #[sea_orm(column_name = "userId")]
-    pub user_id: String,
+    pub user_id: Option<String>,
     #[sea_orm(column_name = "appId")]
     pub app_id: Option<String>,
     #[sea_orm(column_name = "lastUsedAt")]
@@ -43,6 +43,8 @@ pub enum Relation {
     App,
     #[sea_orm(has_many = "super::notification::Entity")]
     Notification,
+    #[sea_orm(has_many = "super::sw_subscription::Entity")]
+    SwSubscription,
     #[sea_orm(
         belongs_to = "super::user::Entity",
         from = "Column::UserId",
@@ -65,6 +67,12 @@ impl Related<super::notification::Entity> for Entity {
     }
 }
 
+impl Related<super::sw_subscription::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::SwSubscription.def()
+    }
+}
+
 impl Related<super::user::Entity> for Entity {
     fn to() -> RelationDef {
         Relation::User.def()

packages/backend-rs/src/model/entity/sw_subscription.rs

@@ -21,10 +21,22 @@ pub struct Model {
     pub publickey: String,
     #[sea_orm(column_name = "sendReadMessage")]
     pub send_read_message: bool,
+    #[sea_orm(column_name = "appAccessTokenId")]
+    pub app_access_token_id: Option<String>,
+    #[sea_orm(column_name = "subscriptionTypes")]
+    pub subscription_types: Vec<String>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
 pub enum Relation {
+    #[sea_orm(
+        belongs_to = "super::access_token::Entity",
+        from = "Column::AppAccessTokenId",
+        to = "super::access_token::Column::Id",
+        on_update = "NoAction",
+        on_delete = "Cascade"
+    )]
+    AccessToken,
     #[sea_orm(
         belongs_to = "super::user::Entity",
         from = "Column::UserId",
@@ -35,6 +47,12 @@ pub enum Relation {
     User,
 }
 
+impl Related<super::access_token::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::AccessToken.def()
+    }
+}
+
 impl Related<super::user::Entity> for Entity {
     fn to() -> RelationDef {
         Relation::User.def()

packages/backend-rs/src/model/entity/user_profile.rs

@@ -70,11 +70,13 @@ pub struct Model {
     pub is_indexable: bool,
     #[sea_orm(column_name = "mutedPatterns")]
     pub muted_patterns: Vec<String>,
+    #[sea_orm(column_type = "JsonBinary")]
+    pub mentions: Json,
+    pub lang: Option<String>,
     #[sea_orm(column_name = "mutedInstances")]
     pub muted_instances: Vec<String>,
     #[sea_orm(column_name = "mutedWords")]
     pub muted_words: Vec<String>,
-    pub lang: Option<String>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

packages/backend-rs/src/service/push_notification.rs

@@ -1,7 +1,7 @@
 use crate::database::db_conn;
 use crate::misc::get_note_summary::{get_note_summary, NoteLike};
 use crate::misc::meta::fetch_meta;
-use crate::model::entity::sw_subscription;
+use crate::model::entity::{access_token, sw_subscription};
 use crate::util::http_client;
 use once_cell::sync::OnceCell;
 use sea_orm::{prelude::*, DbErr};
@@ -101,6 +101,44 @@ fn compact_content(
     Ok(serde_json::from_value(Json::Object(object.clone()))?)
 }
 
+async fn encode_mastodon_payload(
+    mut content: serde_json::Value,
+    db: &DatabaseConnection,
+    subscription: &sw_subscription::Model,
+) -> Result<String, Error> {
+    if !content.is_object() {
+        return Err(Error::InvalidContent("not a JSON object".to_string()));
+    }
+
+    if subscription.app_access_token_id.is_none() {
+        return Err(Error::InvalidContent("no access token".to_string()));
+    }
+
+    let token_id = subscription.app_access_token_id.as_ref().unwrap();
+    let maybe_token = access_token::Entity::find()
+        .filter(access_token::Column::Id.eq(token_id))
+        .one(db)
+        .await?;
+
+    if maybe_token.is_none() {
+        return Err(Error::InvalidContent("access token not found".to_string()));
+    }
+
+    let token = maybe_token.unwrap();
+
+    if token.app_id.is_none() {
+        return Err(Error::InvalidContent("no app ID".to_string()));
+    }
+
+    let object = content.as_object_mut().unwrap();
+    object.insert(
+        "access_token".to_string(),
+        serde_json::to_value(token.token)?,
+    );
+
+    Ok(serde_json::to_string(&content)?)
+}
+
 async fn handle_web_push_failure(
     db: &DatabaseConnection,
     err: WebPushError,
@@ -160,9 +198,9 @@ pub async fn send_push_notification(
         .await?;
 
     // TODO: refactoring
-    let payload = if kind == PushNotificationKind::Mastodon {
-        // Leave the `content` as it is
-        serde_json::to_string(content)?
+    let mut payload = if kind == PushNotificationKind::Mastodon {
+        // Content generated per subscription
+        "".to_string()
     } else {
         // Format the `content` passed from the TypeScript backend
         // for Firefish push notifications
@@ -195,6 +233,15 @@ pub async fn send_push_notification(
             continue;
         }
 
+        if kind == PushNotificationKind::Mastodon {
+            if subscription.app_access_token_id.is_none() {
+                continue;
+            }
+            payload = encode_mastodon_payload(content.clone(), db, subscription).await?;
+        } else if subscription.app_access_token_id.is_some() {
+            continue;
+        }
+
         let subscription_info = SubscriptionInfo {
             endpoint: subscription.endpoint.to_owned(),
             keys: SubscriptionKeys {

packages/backend/package.json

@@ -36,6 +36,8 @@
 		"adm-zip": "0.5.10",
 		"ajv": "8.13.0",
 		"archiver": "7.0.1",
+		"async-lock": "1.4.0",
+		"async-mutex": "^0.5.0",
 		"aws-sdk": "2.1626.0",
 		"axios": "1.7.2",
 		"backend-rs": "workspace:*",
@@ -78,7 +80,6 @@
 		"koa-remove-trailing-slashes": "2.0.3",
 		"koa-send": "5.0.1",
 		"koa-slow": "2.1.0",
-		"megalodon": "workspace:*",
 		"mfm-js": "0.24.0",
 		"mime-types": "2.1.35",
 		"msgpackr": "1.10.2",
@@ -116,6 +117,7 @@
 		"tmp": "0.2.3",
 		"typeorm": "0.3.20",
 		"ulid": "2.3.0",
+		"unfurl.js": "^6.4.0",
 		"uuid": "9.0.1",
 		"websocket": "1.0.35",
 		"xev": "3.0.2"
@@ -124,6 +126,7 @@
 		"@swc/cli": "0.3.12",
 		"@swc/core": "1.5.7",
 		"@types/adm-zip": "0.5.5",
+		"@types/async-lock": "1.4.0",
 		"@types/color-convert": "2.0.3",
 		"@types/content-disposition": "0.5.8",
 		"@types/escape-regexp": "0.0.3",
@@ -164,7 +167,6 @@
 		"@types/tinycolor2": "1.4.6",
 		"@types/tmp": "0.2.6",
 		"@types/uuid": "9.0.8",
-		"@types/web-push": "3.6.3",
 		"@types/websocket": "1.0.10",
 		"@types/ws": "8.5.10",
 		"cross-env": "7.0.3",

packages/backend/src/migration/1709395223612-swSubscriptionAccessToken.ts

@@ -0,0 +1,28 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class SwSubscriptionAccessToken1709395223612
+	implements MigrationInterface
+{
+	name = "SwSubscriptionAccessToken1709395223612";
+
+	async up(queryRunner: QueryRunner) {
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" ADD COLUMN IF NOT EXISTS "appAccessTokenId" character varying(32)`,
+		);
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" ADD CONSTRAINT "FK_98a1aa2db2a5253924f42f38767" FOREIGN KEY ("appAccessTokenId") REFERENCES "access_token"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+		);
+		await queryRunner.query(
+			`CREATE INDEX "IDX_98a1aa2db2a5253924f42f3876" ON "sw_subscription" ("appAccessTokenId") `,
+		);
+	}
+
+	async down(queryRunner: QueryRunner) {
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" DROP CONSTRAINT "FK_98a1aa2db2a5253924f42f38767"`,
+		);
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" DROP COLUMN "appAccessTokenId"`,
+		);
+	}
+}

packages/backend/src/migration/1711075007936-userProfileMentions.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class UserProfileMentions1711075007936 implements MigrationInterface {
+	name = "UserProfileMentions1711075007936";
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "user_profile" ADD "mentions" jsonb NOT NULL DEFAULT '[]'`,
+		);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "user_profile" DROP COLUMN "mentions"`,
+		);
+	}
+}

packages/backend/src/migration/1713108561474-clientCredentials.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class ClientCredentials1713108561474 implements MigrationInterface {
+	name = "ClientCredentials1713108561474";
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "access_token" ALTER COLUMN "userId" DROP NOT NULL`,
+		);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "access_token" ALTER COLUMN "userId" SET NOT NULL`,
+		);
+	}
+}

packages/backend/src/migration/1715181461692-addMastodonSubscriptionType.ts

@@ -0,0 +1,24 @@
+import type { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddMastodonSubscriptionType1715181461692
+	implements MigrationInterface
+{
+	name = "AddMastodonSubscriptionType1715181461692";
+
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" ADD "subscriptionTypes" character varying(64) array NOT NULL DEFAULT '{}'`,
+		);
+		await queryRunner.query(`
+            UPDATE "sw_subscription"
+            SET "subscriptionTypes" = ARRAY['mention', 'status', 'reblog', 'follow', 'follow_request', 'favourite', 'poll', 'update']
+            WHERE "appAccessTokenId" IS NOT NULL;
+        `);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`ALTER TABLE "sw_subscription" DROP COLUMN "subscriptionTypes"`,
+		);
+	}
+}

packages/backend/src/misc/is-filtered.ts

@@ -0,0 +1,36 @@
+import type { User } from "@/models/entities/user.js";
+import type { Note } from "@/models/entities/note.js";
+import type { UserProfile } from "@/models/entities/user-profile.js";
+import { checkWordMute } from "backend-rs";
+import { Cache } from "@/misc/cache.js";
+import { UserProfiles } from "@/models/index.js";
+
+const filteredNoteCache = new Cache<boolean>("filteredNote", 60 * 60 * 24);
+const mutedWordsCache = new Cache<
+	Pick<UserProfile, "mutedWords" | "mutedPatterns">
+>("mutedWords", 60 * 5);
+
+export async function isFiltered(
+	note: Note,
+	user: { id: User["id"] } | null | undefined,
+): Promise<boolean> {
+	if (!user) return false;
+	const profile = await mutedWordsCache.fetch(user.id, () =>
+		UserProfiles.findOneBy({ userId: user.id }).then((p) => ({
+			mutedWords: p?.mutedWords ?? [],
+			mutedPatterns: p?.mutedPatterns ?? [],
+		})),
+	);
+
+	if (
+		!profile ||
+		(profile.mutedPatterns.length < 1 && profile.mutedWords.length < 1)
+	)
+		return false;
+	const ts = (note.updatedAt ?? note.createdAt) as Date | string;
+	const identifier =
+		(typeof ts === "string" ? new Date(ts) : ts)?.getTime() ?? "0";
+	return filteredNoteCache.fetch(`${note.id}:${identifier}:${user.id}`, () =>
+		checkWordMute(note, profile.mutedWords, profile.mutedPatterns),
+	);
+}

packages/backend/src/misc/is-instance-muted.ts

@@ -1,7 +1,12 @@
 import type { Packed } from "./schema.js";
 
+type NoteWithUserHost = { user: { host: string | null } | null };
+
 export function isInstanceMuted(
-	note: Packed<"Note">,
+	note: NoteWithUserHost & {
+		reply: NoteWithUserHost | null;
+		renote: NoteWithUserHost | null;
+	},
 	mutedInstances: Set<string>,
 ): boolean {
 	if (mutedInstances.has(note?.user?.host ?? "")) return true;

packages/backend/src/misc/is-quote.ts

@@ -0,0 +1,10 @@
+import type { Note } from "@/models/entities/note.js";
+
+export default function (note: Note): boolean {
+	return (
+		note.renoteId != null &&
+		(note.text != null ||
+			note.hasPoll ||
+			(note.fileIds != null && note.fileIds.length > 0))
+	);
+}

packages/backend/src/misc/populate-emojis.ts

@@ -14,7 +14,7 @@ const cache = new Cache<Emoji | null>("populateEmojis", 60 * 60 * 12);
 /**
  * 添付用絵文字情報
  */
-type PopulatedEmoji = {
+export type PopulatedEmoji = {
 	name: string;
 	url: string;
 	width: number | null;

packages/backend/src/misc/sql-regex-escape.ts

@@ -0,0 +1,3 @@
+export function sqlRegexEscape(s: string) {
+	return s.replace(/([!$()*+.:<=>?[\\\]^{|}-])/g, "\\$1");
+}

packages/backend/src/models/entities/access-token.ts

@@ -46,8 +46,11 @@ export class AccessToken {
 	public hash: string;
 
 	@Index()
-	@Column(id())
-	public userId: User["id"];
+	@Column({
+		...id(),
+		nullable: true,
+	})
+	public userId: User["id"] | null;
 
 	@Column({
 		...id(),

packages/backend/src/models/entities/note.ts

@@ -307,9 +307,11 @@ export class Note {
 	}
 }
 
-export type IMentionedRemoteUsers = {
+export type IMentionedRemoteUser = {
 	uri: string;
 	url?: string;
 	username: string;
 	host: string;
-}[];
+};
+
+export type IMentionedRemoteUsers = IMentionedRemoteUser[];

packages/backend/src/models/entities/sw-subscription.ts

@@ -9,6 +9,7 @@ import {
 } from "typeorm";
 import { User } from "./user.js";
 import { id } from "../id.js";
+import { AccessToken } from "./access-token.js";
 
 @Entity()
 export class SwSubscription {
@@ -48,5 +49,32 @@ export class SwSubscription {
 	})
 	@JoinColumn()
 	public user: Relation<User>;
+
+	/**
+	 * Type of subscription, used for Mastodon API notifications.
+	 * Empty for Misskey notifications.
+	 */
+	@Column("varchar", {
+		length: 64,
+		array: true,
+		default: "{}",
+	})
+	public subscriptionTypes: string[];
+
+	/**
+	 * App notification app, used for Mastodon API notifications
+	 */
+	@Index()
+	@Column({
+		...id(),
+		nullable: true,
+	})
+	public appAccessTokenId: AccessToken["id"] | null;
+
+	@ManyToOne((type) => AccessToken, {
+		onDelete: "CASCADE",
+	})
+	@JoinColumn()
+	public appAccessToken: AccessToken | null;
 	//#endregion
 }

packages/backend/src/models/entities/user-profile.ts

@@ -11,6 +11,7 @@ import { ffVisibility, notificationTypes } from "@/types.js";
 import { id } from "../id.js";
 import { User } from "./user.js";
 import { Page } from "./page.js";
+import type { IMentionedRemoteUsers } from "./note.js";
 
 // TODO: このテーブルで管理している情報すべてレジストリで管理するようにしても良いかも
 //       ただ、「emailVerified が true なユーザーを find する」のようなクエリは書けなくなるからウーン
@@ -50,6 +51,11 @@ export class UserProfile {
 		verified?: boolean;
 	}[];
 
+	@Column("jsonb", {
+		default: [],
+	})
+	public mentions: IMentionedRemoteUsers;
+
 	@Column("varchar", {
 		length: 32,
 		nullable: true,

packages/backend/src/models/index.ts

@@ -37,7 +37,6 @@ import { AppRepository } from "./repositories/app.js";
 import { FollowingRepository } from "./repositories/following.js";
 import { AbuseUserReportRepository } from "./repositories/abuse-user-report.js";
 import { AuthSessionRepository } from "./repositories/auth-session.js";
-import { UserProfile } from "./entities/user-profile.js";
 import { AttestationChallenge } from "./entities/attestation-challenge.js";
 import { UserSecurityKey } from "./entities/user-security-key.js";
 import { HashtagRepository } from "./repositories/hashtag.js";
@@ -67,6 +66,7 @@ import { Webhook } from "./entities/webhook.js";
 import { UserIp } from "./entities/user-ip.js";
 import { NoteFileRepository } from "./repositories/note-file.js";
 import { NoteEditRepository } from "./repositories/note-edit.js";
+import { UserProfileRepository } from "./repositories/user-profile.js";
 import { ScheduledNote } from "./entities/scheduled-note.js";
 
 export const Announcements = db.getRepository(Announcement);
@@ -83,7 +83,7 @@ export const NoteUnreads = db.getRepository(NoteUnread);
 export const Polls = db.getRepository(Poll);
 export const PollVotes = db.getRepository(PollVote);
 export const Users = UserRepository;
-export const UserProfiles = db.getRepository(UserProfile);
+export const UserProfiles = UserProfileRepository;
 export const UserKeypairs = db.getRepository(UserKeypair);
 export const UserPendings = db.getRepository(UserPending);
 export const AttestationChallenges = db.getRepository(AttestationChallenge);

packages/backend/src/models/repositories/drive-file.ts

@@ -8,6 +8,7 @@ import { config } from "@/config.js";
 import { query, appendQuery } from "@/prelude/url.js";
 import { Users, DriveFolders } from "../index.js";
 import { deepClone } from "@/misc/clone.js";
+import { fetchMeta } from "backend-rs";
 
 type PackOptions = {
 	detail?: boolean;
@@ -221,4 +222,33 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
 		);
 		return items.filter((x): x is Packed<"DriveFile"> => x != null);
 	},
+
+	async getFinalUrl(url: string): Promise<string> {
+		if (!config.proxyRemoteFiles) return url;
+		if (!url.startsWith("https://") && !url.startsWith("http://")) return url;
+		if (url.startsWith(`${config.url}/files`)) return url;
+		if (url.startsWith(`${config.url}/static-assets`)) return url;
+		if (url.startsWith(`${config.url}/identicon`)) return url;
+		if (url.startsWith(`${config.url}/avatar`)) return url;
+
+		const meta = await fetchMeta(true);
+		const baseUrl = meta
+			? meta.objectStorageBaseUrl ??
+				`${meta.objectStorageUseSsl ? "https" : "http"}://${
+					meta.objectStorageEndpoint
+				}${meta.objectStoragePort ? `:${meta.objectStoragePort}` : ""}/${
+					meta.objectStorageBucket
+				}`
+			: null;
+		if (baseUrl !== null && url.startsWith(baseUrl)) return url;
+
+		return `${config.url}/proxy/${encodeURIComponent(
+			new URL(url).pathname,
+		)}?${query({ url: url })}`;
+	},
+
+	async getFinalUrlMaybe(url?: string | null): Promise<string | null> {
+		if (url == null) return null;
+		return this.getFinalUrl(url);
+	},
 });

packages/backend/src/models/repositories/note.ts

@@ -10,6 +10,7 @@ import {
 	Followings,
 	Polls,
 	Channels,
+	UserProfiles,
 	Notes,
 	ScheduledNotes,
 } from "../index.js";
@@ -24,6 +25,7 @@ import {
 } from "@/misc/populate-emojis.js";
 import { db } from "@/db/postgre.js";
 import { IdentifiableError } from "@/misc/identifiable-error.js";
+import { config } from "@/config.js";
 
 export async function populatePoll(note: Note, meId: User["id"] | null) {
 	const poll = await Polls.findOneByOrFail({ noteId: note.id });
@@ -146,6 +148,29 @@ export const NoteRepository = db.getRepository(Note).extend({
 		return true;
 	},
 
+	async mentionedRemoteUsers(note: Note): Promise<string | undefined> {
+		if (note.mentions?.length) {
+			const mentionedUserIds = [...new Set(note.mentions)].sort();
+			const mentionedUsers = await Users.findBy({
+				id: In(mentionedUserIds),
+			});
+			const userProfiles = await UserProfiles.findBy({
+				userId: In(mentionedUserIds),
+			});
+			return JSON.stringify(
+				mentionedUsers.map((u) => ({
+					username: u.username,
+					host: u.host ?? config.host,
+					uri: u.uri ?? `${config.url}/users/${u.id}`,
+					url:
+						userProfiles.find((p) => p.userId === u.id)?.url ??
+						`${config.url}/@${u.username}`,
+				})),
+			);
+		}
+		return undefined;
+	},
+
 	async pack(
 		src: Note["id"] | Note,
 		me?: { id: User["id"] } | null | undefined,
@@ -287,6 +312,7 @@ export const NoteRepository = db.getRepository(Note).extend({
 					}
 				: {}),
 			lang: note.lang,
+			mentionedRemoteUsers: this.mentionedRemoteUsers(note),
 		});
 
 		if (packed.user.isCat && packed.user.speakAsCat && packed.text) {

packages/backend/src/models/repositories/user-profile.ts

@@ -0,0 +1,87 @@
+import { db } from "@/db/postgre.js";
+import { UserProfile } from "@/models/entities/user-profile.js";
+import mfm from "mfm-js";
+import { extractMentions } from "@/misc/extract-mentions.js";
+import {
+	type ProfileMention,
+	resolveMentionToUserAndProfile,
+} from "@/remote/resolve-user.js";
+import type {
+	IMentionedRemoteUser,
+	IMentionedRemoteUsers,
+} from "@/models/entities/note.js";
+import { unique } from "@/prelude/array.js";
+import { config } from "@/config.js";
+import { Mutex, Semaphore } from "async-mutex";
+
+const queue = new Semaphore(5);
+
+export const UserProfileRepository = db.getRepository(UserProfile).extend({
+	// We must never await this without promiseEarlyReturn, otherwise giant webring-style profile mention trees will cause the queue to stop working
+	async updateMentions(
+		id: UserProfile["userId"],
+		limiter: RecursionLimiter = new RecursionLimiter(),
+	) {
+		const profile = await this.findOneBy({ userId: id });
+		if (!profile) return;
+		const tokens: mfm.MfmNode[] = [];
+
+		if (profile.description) tokens.push(...mfm.parse(profile.description));
+		if (profile.fields.length > 0)
+			tokens.push(
+				...profile.fields.flatMap((p) =>
+					mfm.parse(p.value).concat(mfm.parse(p.name)),
+				),
+			);
+
+		return queue.runExclusive(async () => {
+			const partial = {
+				mentions: await populateMentions(tokens, profile.userHost, limiter),
+			};
+			return UserProfileRepository.update(profile.userId, partial);
+		});
+	},
+});
+
+async function populateMentions(
+	tokens: mfm.MfmNode[],
+	objectHost: string | null,
+	limiter: RecursionLimiter,
+): Promise<IMentionedRemoteUsers> {
+	const mentions = extractMentions(tokens);
+	const resolved = await Promise.all(
+		mentions.map((m) =>
+			resolveMentionToUserAndProfile(m.username, m.host, objectHost, limiter),
+		),
+	);
+	const remote = resolved.filter(
+		(p): p is ProfileMention =>
+			!!p &&
+			p.data.host !== config.host &&
+			(p.data.host !== null || objectHost !== null),
+	);
+	const res = remote.map((m) => {
+		return {
+			uri: m.user.uri,
+			url: m.profile?.url ?? undefined,
+			username: m.data.username,
+			host: m.data.host,
+		} as IMentionedRemoteUser;
+	});
+
+	return unique(res);
+}
+
+export class RecursionLimiter {
+	private counter;
+	private mutex = new Mutex();
+	constructor(count = 10) {
+		this.counter = count;
+	}
+
+	public shouldContinue(): Promise<boolean> {
+		return this.mutex.runExclusive(() => {
+			return this.counter-- > 0;
+		});
+	}
+}

packages/backend/src/prelude/array.ts

@@ -51,6 +51,17 @@ export function unique<T>(xs: T[]): T[] {
 	return [...new Set(xs)];
 }
 
+/**
+ * Filters an array of elements based on unique outputs of a key function
+ */
+export function uniqBy<T, U>(a: T[], key: (elm: T) => U): T[] {
+	const seen = new Set<U>();
+	return a.filter((item) => {
+		const k = key(item);
+		return seen.has(k) ? false : seen.add(k);
+	});
+}
+
 export function sum(xs: number[]): number {
 	return xs.reduce((a, b) => a + b, 0);
 }
@@ -150,3 +161,7 @@ export function toArray<T>(x: T | T[] | undefined): T[] {
 export function toSingle<T>(x: T | T[] | undefined): T | undefined {
 	return Array.isArray(x) ? x[0] : x;
 }
+
+export function toSingleLast<T>(x: T | T[] | undefined): T | undefined {
+	return Array.isArray(x) ? x.at(-1) : x;
+}

packages/backend/src/prelude/await-all.ts

@@ -4,7 +4,27 @@ export type Promiseable<T> = {
 	[K in keyof T]: Promise<T[K]> | T[K];
 };
 
-export async function awaitAll<T>(obj: Promiseable<T>): Promise<T> {
+type RecursiveResolvePromise<U> = U extends Date
+	? U
+	: U extends Array<infer V>
+		? Array<ResolvedPromise<V>>
+		: U extends object
+			? { [key in keyof U]: ResolvedPromise<U[key]> }
+			: U;
+
+type ResolvedPromise<T> = T extends Promise<infer U>
+	? RecursiveResolvePromise<U>
+	: RecursiveResolvePromise<T>;
+
+export type OuterPromise<T> = Promise<{
+	[K in keyof T]: ResolvedPromise<T[K]>;
+}>;
+
+/**
+ * Resolve all promises in the object recursively,
+ * and return a promise that resolves to the object with all promises resolved.
+ */
+export async function awaitAll<T>(obj: Promiseable<T>): OuterPromise<T> {
 	const target = {} as T;
 	const keys = unsafeCast<(keyof T)[]>(Object.keys(obj));
 	const values = Object.values(obj) as any[];
@@ -21,5 +41,5 @@ export async function awaitAll<T>(obj: Promiseable<T>): Promise<T> {
 		target[keys[i]] = resolvedValues[i];
 	}
 
-	return target;
+	return target as OuterPromise<T>;
 }

packages/backend/src/prelude/promise.ts

@@ -0,0 +1,13 @@
+/**
+ * Returns T if promise settles before timeout,
+ * otherwise returns void, finishing execution in the background.
+ */
+export async function promiseEarlyReturn<T>(
+	promise: Promise<T>,
+	after: number,
+): Promise<T | void> {
+	const timer: Promise<void> = new Promise((res) =>
+		setTimeout(() => res(undefined), after),
+	);
+	return Promise.race([promise, timer]);
+}

packages/backend/src/remote/resolve-user.ts

@@ -3,21 +3,53 @@ import chalk from "chalk";
 import { IsNull } from "typeorm";
 import { config } from "@/config.js";
 import type { User, IRemoteUser } from "@/models/entities/user.js";
-import { Users } from "@/models/index.js";
+import { Cache } from "@/misc/cache.js";
+import { UserProfiles, Users } from "@/models/index.js";
 import { toPuny } from "backend-rs";
 import webFinger from "./webfinger.js";
 import { createPerson, updatePerson } from "./activitypub/models/person.js";
 import { remoteLogger } from "./logger.js";
 import { inspect } from "node:util";
+import type { UserProfile } from "@/models/entities/user-profile.js";
+import { RecursionLimiter } from "@/models/repositories/user-profile.js";
+import { promiseEarlyReturn } from "@/prelude/promise.js";
+import type { IMentionedRemoteUsers } from "@/models/entities/note.js";
 
 const logger = remoteLogger.createSubLogger("resolve-user");
+const localUsernameCache = new Cache<string | null>(
+	"localUserNameCapitalization",
+	60 * 60 * 24,
+);
+const profileMentionCache = new Cache<ProfileMention | null>(
+	"resolveProfileMentions",
+	60 * 60,
+);
+
+export type ProfileMention = {
+	user: User;
+	profile: UserProfile | null;
+	data: {
+		username: string;
+		host: string | null;
+	};
+};
+
+type refreshType =
+	| "refresh"
+	| "refresh-in-background"
+	| "refresh-timeout-1500ms"
+	| "no-refresh";
 
 export async function resolveUser(
 	username: string,
 	host: string | null,
+	refresh: refreshType = "refresh",
+	limiter: RecursionLimiter = new RecursionLimiter(),
 ): Promise<User> {
 	const usernameLower = username.toLowerCase();
 
+	// Return local user if host part is empty
+
 	if (host == null) {
 		logger.info(`return local user: ${usernameLower}`);
 		return await Users.findOneBy({ usernameLower, host: IsNull() }).then(
@@ -33,7 +65,9 @@ export async function resolveUser(
 
 	host = toPuny(host);
 
-	if (config.host === host) {
+	// Also return local user if host part is specified but referencing the local instance
+
+	if (config.host === host || config.host === host) {
 		logger.info(`return local user: ${usernameLower}`);
 		return await Users.findOneBy({ usernameLower, host: IsNull() }).then(
 			(u) => {
@@ -46,24 +80,63 @@ export async function resolveUser(
 		);
 	}
 
-	const user = (await Users.findOneBy({
+	// Check if remote user is already in the database
+
+	let user = (await Users.findOneBy({
 		usernameLower,
 		host,
 	})) as IRemoteUser | null;
 
 	const acctLower = `${usernameLower}@${host}`;
 
-	if (user == null) {
-		const self = await resolveSelf(acctLower);
+	// If not, look up the user on the remote server
 
-		logger.info(`return new remote user: ${chalk.magenta(acctLower)}`);
-		return await createPerson(self.href);
+	if (user == null) {
+		// Run WebFinger
+		const fingerRes = await resolveUserWebFinger(acctLower);
+		const finalAcct = subjectToAcct(fingerRes.subject);
+		const finalAcctLower = finalAcct.toLowerCase();
+		const m = finalAcct.match(/^([^@]+)@(.*)/);
+		const subjectHost = m ? m[2] : undefined;
+
+		// If subject is different, we're dealing with a split domain setup (that's already been validated by resolveUserWebFinger)
+		if (acctLower !== finalAcctLower) {
+			logger.info("re-resolving split domain redirect user...");
+			const m = finalAcct.match(/^([^@]+)@(.*)/);
+			if (m) {
+				// Re-check if we already have the user in the database post-redirect
+				user = (await Users.findOneBy({
+					usernameLower: usernameLower,
+					host: subjectHost,
+				})) as IRemoteUser | null;
+
+				// If yes, return existing user
+				if (user != null) {
+					logger.info(
+						`return existing remote user: ${chalk.magenta(finalAcctLower)}`,
+					);
+					return user;
+				}
+				// Otherwise create and return new user
+				else {
+					logger.info(
+						`return new remote user: ${chalk.magenta(finalAcctLower)}`,
+					);
+					return await createPerson(fingerRes.self.href);
+				}
+			}
+		}
+
+		// Not a split domain setup, so we can simply create and return the new user
+		logger.info(`return new remote user: ${chalk.magenta(finalAcctLower)}`);
+		return await createPerson(fingerRes.self.href);
 	}
 
-	// If user information is out of date, return it by starting over from WebFilger
+	// If user information is out of date, return it by starting over from WebFinger
 	if (
-		user.lastFetchedAt == null ||
-		Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24
+		(refresh === "refresh" || refresh === "refresh-timeout-1500ms") &&
+		(user.lastFetchedAt == null ||
+			Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24)
 	) {
 		// Prevent multiple attempts to connect to unconnected instances, update before each attempt to prevent subsequent similar attempts
 		await Users.update(user.id, {
@@ -71,17 +144,17 @@ export async function resolveUser(
 		});
 
 		logger.info(`try resync: ${acctLower}`);
-		const self = await resolveSelf(acctLower);
+		const fingerRes = await resolveUserWebFinger(acctLower);
 
-		if (user.uri !== self.href) {
+		if (user.uri !== fingerRes.self.href) {
 			// if uri mismatch, Fix (user@host <=> AP's Person id(IRemoteUser.uri)) mapping.
 			logger.info(`uri missmatch: ${acctLower}`);
 			logger.info(
-				`recovery missmatch uri for (username=${username}, host=${host}) from ${user.uri} to ${self.href}`,
+				`recovery mismatch uri for (username=${username}, host=${host}) from ${user.uri} to ${fingerRes.self.href}`,
 			);
 
 			// validate uri
-			const uri = new URL(self.href);
+			const uri = new URL(fingerRes.self.href);
 			if (uri.hostname !== host) {
 				throw new Error("Invalid uri");
 			}
@@ -92,23 +165,60 @@ export async function resolveUser(
 					host: host,
 				},
 				{
-					uri: self.href,
+					uri: fingerRes.self.href,
 				},
 			);
 		} else {
 			logger.info(`uri is fine: ${acctLower}`);
 		}
 
-		await updatePerson(self.href);
+		const finalAcct = subjectToAcct(fingerRes.subject);
+		const finalAcctLower = finalAcct.toLowerCase();
+		const m = finalAcct.match(/^([^@]+)@(.*)/);
+		const finalHost = m ? m[2] : null;
 
-		logger.info(`return resynced remote user: ${acctLower}`);
-		return await Users.findOneBy({ uri: self.href }).then((u) => {
+		// Update user.host if we're dealing with an account that's part of a split domain setup that hasn't been fixed yet
+		if (m && user.host !== finalHost) {
+			logger.info(
+				`updating user host to subject acct host: ${user.host} -> ${finalHost}`,
+			);
+			await Users.update(
+				{
+					usernameLower,
+					host: user.host,
+				},
+				{
+					host: finalHost,
+				},
+			);
+		}
+
+		if (refresh === "refresh") {
+			await updatePerson(fingerRes.self.href);
+			logger.info(`return resynced remote user: ${finalAcctLower}`);
+		} else if (refresh === "refresh-timeout-1500ms") {
+			const res = await promiseEarlyReturn(
+				updatePerson(fingerRes.self.href),
+				1500,
+			);
+			logger.info(`return possibly resynced remote user: ${finalAcctLower}`);
+		}
+
+		return await Users.findOneBy({ uri: fingerRes.self.href }).then((u) => {
 			if (u == null) {
 				throw new Error("user not found");
 			} else {
 				return u;
 			}
 		});
+	} else if (
+		refresh === "refresh-in-background" &&
+		(user.lastFetchedAt == null ||
+			Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24)
+	) {
+		// Run the refresh in the background
+		// noinspection ES6MissingAwait
+		resolveUser(username, host, "refresh", limiter);
 	}
 
 	logger.info(`return existing remote user: ${acctLower}`);
@@ -136,3 +246,185 @@ async function resolveSelf(acctLower: string) {
 	}
 	return self;
 }
+
+async function getLocalUsernameCached(
+	username: string,
+): Promise<string | null> {
+	return localUsernameCache.fetch(username.toLowerCase(), () =>
+		Users.findOneBy({
+			usernameLower: username.toLowerCase(),
+			host: IsNull(),
+		}).then((p) => (p ? p.username : null)),
+	);
+}
+
+export function getMentionFallbackUri(
+	username: string,
+	host: string | null,
+	objectHost: string | null,
+): string {
+	let fallback = `${config.url}/@${username}`;
+	if (host !== null && host !== config.host) fallback += `@${host}`;
+	else if (
+		objectHost !== null &&
+		objectHost !== config.host &&
+		host !== config.host
+	)
+		fallback += `@${objectHost}`;
+
+	return fallback;
+}
+
+export async function resolveMentionFromCache(
+	username: string,
+	host: string | null,
+	objectHost: string | null,
+	cache: IMentionedRemoteUsers,
+): Promise<{
+	username: string;
+	href: string;
+	host: string;
+	isLocal: boolean;
+} | null> {
+	const isLocal =
+		(host === null && objectHost === null) || host === config.host;
+	if (isLocal) {
+		const finalUsername = await getLocalUsernameCached(username);
+		if (finalUsername === null) return null;
+		username = finalUsername;
+	}
+
+	const fallback = getMentionFallbackUri(username, host, objectHost);
+	const cached = cache.find(
+		(r) =>
+			r.username.toLowerCase() === username.toLowerCase() &&
+			r.host === (host ?? objectHost),
+	);
+	const href = cached?.url ?? cached?.uri;
+	if (cached && href != null)
+		return {
+			username: cached.username,
+			href: href,
+			isLocal,
+			host: cached.host,
+		};
+	if (isLocal)
+		return { username: username, href: fallback, isLocal, host: config.host };
+	return null;
+}
+
+export async function resolveMentionToUserAndProfile(
+	username: string,
+	host: string | null,
+	objectHost: string | null,
+	limiter: RecursionLimiter,
+) {
+	return profileMentionCache.fetch(
+		`${username}@${host ?? objectHost}`,
+		async () => {
+			try {
+				const user = await resolveUser(
+					username,
+					host ?? objectHost,
+					"no-refresh",
+					limiter,
+				);
+				const profile = await UserProfiles.findOneBy({ userId: user.id });
+				const data = { username, host: host ?? objectHost };
+
+				return { user, profile, data };
+			} catch {
+				return null;
+			}
+		},
+	);
+}
+
+async function resolveUserWebFinger(
+	acctLower: string,
+	recurse = true,
+): Promise<{
+	subject: string;
+	self: {
+		href: string;
+		rel?: string;
+	};
+}> {
+	logger.info(`WebFinger for ${chalk.yellow(acctLower)}`);
+	const fingerRes = await webFinger(acctLower).catch((e) => {
+		logger.error(
+			`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${
+				e.statusCode || e.message
+			}`,
+		);
+		throw new Error(
+			`Failed to WebFinger for ${acctLower}: ${e.statusCode || e.message}`,
+		);
+	});
+	const self = fingerRes.links.find(
+		(link) => link.rel != null && link.rel.toLowerCase() === "self",
+	);
+	if (!self) {
+		logger.error(
+			`Failed to WebFinger for ${chalk.yellow(acctLower)}: self link not found`,
+		);
+		throw new Error("self link not found");
+	}
+	if (`${acctToSubject(acctLower)}` !== normalizeSubject(fingerRes.subject)) {
+		logger.info(
+			`acct subject mismatch (${acctToSubject(
+				acctLower,
+			)} !== ${normalizeSubject(
+				fingerRes.subject,
+			)}), possible split domain deployment detected, repeating webfinger`,
+		);
+		if (!recurse) {
+			logger.error(
+				"split domain verification failed (recurse limit reached), aborting",
+			);
+			throw new Error(
+				"split domain verification failed (recurse limit reached), aborting",
+			);
+		}
+		const initialAcct = subjectToAcct(fingerRes.subject);
+		const initialAcctLower = initialAcct.toLowerCase();
+		const splitFingerRes = await resolveUserWebFinger(initialAcctLower, false);
+		const finalAcct = subjectToAcct(splitFingerRes.subject);
+		const finalAcctLower = finalAcct.toLowerCase();
+		if (initialAcct !== finalAcct) {
+			logger.error(
+				"split domain verification failed (subject mismatch), aborting",
+			);
+			throw new Error(
+				"split domain verification failed (subject mismatch), aborting",
+			);
+		}
+
+		logger.info(
+			`split domain configuration detected: ${acctLower} -> ${finalAcctLower}`,
+		);
+
+		return splitFingerRes;
+	}
+
+	return {
+		subject: fingerRes.subject,
+		self: self,
+	};
+}
+
+function subjectToAcct(subject: string): string {
+	if (!subject.startsWith("acct:")) {
+		logger.error("Subject isnt a valid acct");
+		throw "Subject isnt a valid acct";
+	}
+	return subject.substring(5);
+}
+
+function acctToSubject(acct: string): string {
+	return normalizeSubject(`acct:${acct}`);
+}
+
+function normalizeSubject(subject: string): string {
+	return subject.toLowerCase();
+}

packages/backend/src/server/api/common/generate-following-query.ts

@@ -0,0 +1,59 @@
+import { Brackets, type SelectQueryBuilder } from "typeorm";
+import type { User } from "@/models/entities/user.js";
+import { Followings, Notes } from "@/models/index.js";
+import { Cache } from "@/misc/cache.js";
+import { apiLogger } from "@/server/api/logger.js";
+
+const cache = new Cache<number>("homeTlQueryData", 60 * 60 * 24);
+const cutoff = 250; // 250 posts in the last 7 days, constant determined by comparing benchmarks for cutoff values between 100 and 2500
+const logger = apiLogger.createSubLogger("heuristics");
+
+export async function generateFollowingQuery(
+	q: SelectQueryBuilder<any>,
+	me: { id: User["id"] },
+): Promise<void> {
+	const followingQuery = Followings.createQueryBuilder("following")
+		.select("following.followeeId")
+		.where("following.followerId = :meId");
+
+	const heuristic = await cache.fetch(me.id, async () => {
+		const curr = new Date();
+		const prev = new Date();
+		prev.setDate(prev.getDate() - 7);
+		return Notes.createQueryBuilder("note")
+			.where("note.createdAt > :prev", { prev })
+			.andWhere("note.createdAt < :curr", { curr })
+			.andWhere(
+				new Brackets((qb) => {
+					qb.where(`note.userId IN (${followingQuery.getQuery()})`);
+					qb.orWhere("note.userId = :meId", { meId: me.id });
+				}),
+			)
+			.getCount()
+			.then((res) => {
+				logger.info(
+					`Calculating heuristics for user ${me.id} took ${
+						new Date().getTime() - curr.getTime()
+					}ms`,
+				);
+				return res;
+			});
+	});
+
+	const shouldUseUnion = heuristic < cutoff;
+
+	q.andWhere(
+		new Brackets((qb) => {
+			if (shouldUseUnion) {
+				qb.where(
+					`note.userId = ANY(array(${followingQuery.getQuery()} UNION ALL VALUES (:meId)))`,
+				);
+			} else {
+				qb.where("note.userId = :meId");
+				qb.orWhere(`note.userId IN (${followingQuery.getQuery()})`);
+			}
+		}),
+	);
+
+	q.setParameters({ meId: me.id });
+}

packages/backend/src/server/api/common/generate-fts-query.ts

@@ -0,0 +1,431 @@
+import {
+	Brackets,
+	type SelectQueryBuilder,
+	type WhereExpressionBuilder,
+} from "typeorm";
+import { sqlLikeEscape } from "backend-rs";
+import { sqlRegexEscape } from "@/misc/sql-regex-escape.js";
+import {
+	Followings,
+	NoteFavorites,
+	NoteReactions,
+	Users,
+} from "@/models/index.js";
+import type { Note } from "@/models/entities/note";
+import type { Following } from "@/models/entities/following";
+import type { NoteFavorite } from "@/models/entities/note-favorite";
+
+const filters = {
+	from: fromFilter,
+	"-from": fromFilterInverse,
+	mention: mentionFilter,
+	"-mention": mentionFilterInverse,
+	reply: replyFilter,
+	"-reply": replyFilterInverse,
+	to: replyFilter,
+	"-to": replyFilterInverse,
+	before: beforeFilter,
+	until: beforeFilter,
+	after: afterFilter,
+	since: afterFilter,
+	instance: instanceFilter,
+	"-instance": instanceFilterInverse,
+	domain: instanceFilter,
+	"-domain": instanceFilterInverse,
+	host: instanceFilter,
+	"-host": instanceFilterInverse,
+	filter: miscFilter,
+	"-filter": miscFilterInverse,
+	in: inFilter,
+	"-in": inFilterInverse,
+	has: attachmentFilter,
+} as Record<
+	string,
+	(query: SelectQueryBuilder<Note>, search: string, id: number) => void
+>;
+
+export function generateFtsQuery(
+	query: SelectQueryBuilder<Note>,
+	q: string,
+): void {
+	const components = q.trim().split(" ");
+	const terms: string[] = [];
+	const finalTerms: string[] = [];
+	let counter = 0;
+	let caseSensitive = false;
+	let matchWords = false;
+
+	for (const component of components) {
+		const split = component.split(":");
+		if (split.length > 1 && filters[split[0]] !== undefined)
+			filters[split[0]](query, split.slice(1).join(":"), counter++);
+		else if (
+			split.length > 1 &&
+			(split[0] === "search" || split[0] === "match")
+		)
+			matchWords = split[1] === "word" || split[1] === "words";
+		else if (split.length > 1 && split[0] === "case")
+			caseSensitive = split[1] === "sensitive";
+		else terms.push(component);
+	}
+
+	let idx = 0;
+	let state: "idle" | "quote" | "parenthesis" = "idle";
+	for (let i = 0; i < terms.length; i++) {
+		if (state === "idle") {
+			if (
+				(terms[i].startsWith('"') && terms[i].endsWith('"')) ||
+				(terms[i].startsWith("(") && terms[i].endsWith(")"))
+			) {
+				finalTerms.push(trimStartAndEnd(terms[i]));
+			} else if (terms[i].startsWith('"')) {
+				idx = i;
+				state = "quote";
+			} else if (terms[i].startsWith("(")) {
+				idx = i;
+				state = "parenthesis";
+			} else {
+				finalTerms.push(terms[i]);
+			}
+		} else if (state === "quote" && terms[i].endsWith('"')) {
+			finalTerms.push(extractToken(terms, idx, i));
+			state = "idle";
+		} else if (state === "parenthesis" && terms[i].endsWith(")")) {
+			query.andWhere(
+				new Brackets((qb) => {
+					for (const term of extractToken(terms, idx, i).split(" OR ")) {
+						const id = counter++;
+						appendSearchQuery(
+							term,
+							"or",
+							query,
+							qb,
+							id,
+							term.startsWith("-"),
+							matchWords,
+							caseSensitive,
+						);
+					}
+				}),
+			);
+			state = "idle";
+		}
+	}
+
+	if (state !== "idle") {
+		finalTerms.push(
+			...extractToken(terms, idx, terms.length - 1, false)
+				.substring(1)
+				.split(" "),
+		);
+	}
+
+	for (const term of finalTerms) {
+		const id = counter++;
+		appendSearchQuery(
+			term,
+			"and",
+			query,
+			query,
+			id,
+			term.startsWith("-"),
+			matchWords,
+			caseSensitive,
+		);
+	}
+}
+
+function fromFilter(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.andWhere(`note.userId = (${userQuery.getQuery()})`);
+	query.setParameters(userQuery.getParameters());
+}
+
+function fromFilterInverse(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.andWhere(`note.userId <> (${userQuery.getQuery()})`);
+	query.setParameters(userQuery.getParameters());
+}
+
+function mentionFilter(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.addCommonTableExpression(userQuery.getQuery(), `cte_${id}`, {
+		materialized: true,
+	});
+	query.andWhere(
+		`note.mentions @> array[(SELECT * FROM cte_${id})]::varchar[]`,
+	);
+	query.setParameters(userQuery.getParameters());
+}
+
+function mentionFilterInverse(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.addCommonTableExpression(userQuery.getQuery(), `cte_${id}`, {
+		materialized: true,
+	});
+	query.andWhere(
+		`NOT (note.mentions @> array[(SELECT * FROM cte_${id})]::varchar[])`,
+	);
+	query.setParameters(userQuery.getParameters());
+}
+
+function replyFilter(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.andWhere(`note.replyUserId = (${userQuery.getQuery()})`);
+	query.setParameters(userQuery.getParameters());
+}
+
+function replyFilterInverse(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	const userQuery = generateUserSubquery(filter, id);
+	query.andWhere(`note.replyUserId <> (${userQuery.getQuery()})`);
+	query.setParameters(userQuery.getParameters());
+}
+
+function beforeFilter(query: SelectQueryBuilder<Note>, filter: string) {
+	query.andWhere("note.createdAt < :before", { before: filter });
+}
+
+function afterFilter(query: SelectQueryBuilder<Note>, filter: string) {
+	query.andWhere("note.createdAt > :after", { after: filter });
+}
+
+function instanceFilter(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	if (filter === "local") {
+		query.andWhere("note.userHost IS NULL");
+	} else {
+		query.andWhere(`note.userHost = :instance_${id}`);
+		query.setParameter(`instance_${id}`, filter);
+	}
+}
+
+function instanceFilterInverse(
+	query: SelectQueryBuilder<Note>,
+	filter: string,
+	id: number,
+) {
+	if (filter === "local") {
+		query.andWhere("note.userHost IS NOT NULL");
+	} else {
+		query.andWhere(`note.userHost <> :instance_${id}`);
+		query.setParameter(`instance_${id}`, filter);
+	}
+}
+
+function miscFilter(query: SelectQueryBuilder<Note>, filter: string) {
+	let subQuery: SelectQueryBuilder<Following> | null = null;
+	if (filter === "followers") {
+		subQuery = Followings.createQueryBuilder("following")
+			.select("following.followerId")
+			.where("following.followeeId = :meId");
+	} else if (filter === "following") {
+		subQuery = Followings.createQueryBuilder("following")
+			.select("following.followeeId")
+			.where("following.followerId = :meId");
+	} else if (filter === "replies" || filter === "reply") {
+		query.andWhere("note.replyId IS NOT NULL");
+	} else if (
+		filter === "boosts" ||
+		filter === "boost" ||
+		filter === "renotes" ||
+		filter === "renote"
+	) {
+		query.andWhere("note.renoteId IS NOT NULL");
+	}
+
+	if (subQuery !== null)
+		query.andWhere(`note.userId IN (${subQuery.getQuery()})`);
+}
+
+function miscFilterInverse(query: SelectQueryBuilder<Note>, filter: string) {
+	let subQuery: SelectQueryBuilder<Following> | null = null;
+	if (filter === "followers") {
+		subQuery = Followings.createQueryBuilder("following")
+			.select("following.followerId")
+			.where("following.followeeId = :meId");
+	} else if (filter === "following") {
+		subQuery = Followings.createQueryBuilder("following")
+			.select("following.followeeId")
+			.where("following.followerId = :meId");
+	} else if (filter === "replies" || filter === "reply") {
+		query.andWhere("note.replyId IS NULL");
+	} else if (
+		filter === "boosts" ||
+		filter === "boost" ||
+		filter === "renotes" ||
+		filter === "renote"
+	) {
+		query.andWhere("note.renoteId IS NULL");
+	}
+
+	if (subQuery !== null)
+		query.andWhere(`note.userId NOT IN (${subQuery.getQuery()})`);
+}
+
+function inFilter(query: SelectQueryBuilder<Note>, filter: string) {
+	let subQuery: SelectQueryBuilder<NoteFavorite> | null = null;
+	if (filter === "bookmarks") {
+		subQuery = NoteFavorites.createQueryBuilder("bookmark")
+			.select("bookmark.noteId")
+			.where("bookmark.userId = :meId");
+	} else if (
+		filter === "favorites" ||
+		filter === "favourites" ||
+		filter === "reactions" ||
+		filter === "likes"
+	) {
+		subQuery = NoteReactions.createQueryBuilder("react")
+			.select("react.noteId")
+			.where("react.userId = :meId");
+	}
+
+	if (subQuery !== null) query.andWhere(`note.id IN (${subQuery.getQuery()})`);
+}
+
+function inFilterInverse(query: SelectQueryBuilder<Note>, filter: string) {
+	let subQuery: SelectQueryBuilder<NoteFavorite> | null = null;
+	if (filter === "bookmarks") {
+		subQuery = NoteFavorites.createQueryBuilder("bookmark")
+			.select("bookmark.noteId")
+			.where("bookmark.userId = :meId");
+	} else if (
+		filter === "favorites" ||
+		filter === "favourites" ||
+		filter === "reactions" ||
+		filter === "likes"
+	) {
+		subQuery = NoteReactions.createQueryBuilder("react")
+			.select("react.noteId")
+			.where("react.userId = :meId");
+	}
+
+	if (subQuery !== null)
+		query.andWhere(`note.id NOT IN (${subQuery.getQuery()})`);
+}
+
+function attachmentFilter(query: SelectQueryBuilder<Note>, filter: string) {
+	switch (filter) {
+		case "image":
+			query.andWhere(`note."attachedFileTypes"::varchar ILIKE '%image/%'`);
+			break;
+		case "video":
+			query.andWhere(`note."attachedFileTypes"::varchar ILIKE '%video/%'`);
+			break;
+		case "audio":
+			query.andWhere(`note."attachedFileTypes"::varchar ILIKE '%audio/%'`);
+			break;
+		case "file":
+			query.andWhere(`note."attachedFileTypes" <> '{}'`);
+			query.andWhere(
+				`NOT (note."attachedFileTypes"::varchar ILIKE '%image/%')`,
+			);
+			query.andWhere(
+				`NOT (note."attachedFileTypes"::varchar ILIKE '%video/%')`,
+			);
+			query.andWhere(
+				`NOT (note."attachedFileTypes"::varchar ILIKE '%audio/%')`,
+			);
+			break;
+		default:
+			break;
+	}
+}
+
+function generateUserSubquery(filter: string, id: number) {
+	if (filter.startsWith("@")) filter = filter.substring(1);
+	const split = filter.split("@");
+
+	const query = Users.createQueryBuilder("user")
+		.select("user.id")
+		.where(`user.usernameLower = :user_${id}`)
+		.andWhere(
+			`user.host ${split[1] !== undefined ? `= :host_${id}` : "IS NULL"}`,
+		);
+
+	query.setParameter(`user_${id}`, split[0].toLowerCase());
+
+	if (split[1] !== undefined)
+		query.setParameter(`host_${id}`, split[1].toLowerCase());
+
+	return query;
+}
+
+function extractToken(
+	array: string[],
+	start: number,
+	end: number,
+	trim = true,
+) {
+	const slice = array.slice(start, end + 1).join(" ");
+	return trim ? trimStartAndEnd(slice) : slice;
+}
+
+function trimStartAndEnd(str: string) {
+	return str.substring(1, str.length - 1);
+}
+
+function appendSearchQuery(
+	term: string,
+	mode: "and" | "or",
+	query: SelectQueryBuilder<Note>,
+	qb: SelectQueryBuilder<Note> | WhereExpressionBuilder,
+	id: number,
+	negate: boolean,
+	matchWords: boolean,
+	caseSensitive: boolean,
+) {
+	const sql = `note.text ${getSearchMatchOperator(
+		negate,
+		matchWords,
+		caseSensitive,
+	)} :q_${id}`;
+	if (mode === "and") qb.andWhere(sql);
+	else if (mode === "or") qb.orWhere(sql);
+	query.setParameter(
+		`q_${id}`,
+		escapeSqlSearchParam(term.substring(negate ? 1 : 0), matchWords),
+	);
+}
+
+function getSearchMatchOperator(
+	negate: boolean,
+	matchWords: boolean,
+	caseSensitive: boolean,
+) {
+	return `${negate ? "NOT " : ""}${
+		matchWords ? (caseSensitive ? "~" : "~*") : caseSensitive ? "LIKE" : "ILIKE"
+	}`;
+}
+
+function escapeSqlSearchParam(param: string, matchWords: boolean) {
+	return matchWords
+		? `\\y${sqlRegexEscape(param)}\\y`
+		: `%${sqlLikeEscape(param)}%`;
+}

packages/backend/src/server/api/common/generate-list-query.ts

@@ -0,0 +1,25 @@
+import { Brackets, type SelectQueryBuilder } from "typeorm";
+import type { User } from "@/models/entities/user.js";
+import { UserListJoinings, UserLists } from "@/models/index.js";
+
+export function generateListQuery(
+	q: SelectQueryBuilder<any>,
+	me: { id: User["id"] },
+): void {
+	const listQuery = UserLists.createQueryBuilder("list")
+		.select("list.id")
+		.andWhere("list.userId = :meId");
+
+	const memberQuery = UserListJoinings.createQueryBuilder("member")
+		.select("member.userId")
+		.where(`member.userListId IN (${listQuery.getQuery()})`);
+
+	q.andWhere(
+		new Brackets((qb) => {
+			qb.where("note.userId = :meId");
+			qb.orWhere(`note.userId NOT IN (${memberQuery.getQuery()})`);
+		}),
+	);
+
+	q.setParameters({ meId: me.id });
+}

packages/backend/src/server/api/index.ts

@@ -7,10 +7,7 @@ import Router from "@koa/router";
 import multer from "@koa/multer";
 import bodyParser from "koa-bodyparser";
 import cors from "@koa/cors";
-import {
-	apiMastodonCompatible,
-	getClient,
-} from "./mastodon/ApiMastodonCompatibleService.js";
+import { setupMastodonApi } from "./mastodon/index.js";
 import { AccessTokens, Users } from "@/models/index.js";
 import { config } from "@/config.js";
 import endpoints from "./endpoints.js";
@@ -20,10 +17,6 @@ import signup from "./private/signup.js";
 import signin from "./private/signin.js";
 import signupPending from "./private/signup-pending.js";
 import verifyEmail from "./private/verify-email.js";
-import { koaBody } from "koa-body";
-import { convertAttachment } from "./mastodon/converters.js";
-import { apiLogger } from "./logger.js";
-import { inspect } from "node:util";
 
 // Init app
 const app = new Koa();
@@ -66,64 +59,7 @@ router.use(
 	}),
 );
 
-mastoRouter.use(
-	koaBody({
-		multipart: true,
-		urlencoded: true,
-	}),
-);
-
-mastoFileRouter.post("/v1/media", upload.single("file"), async (ctx) => {
-	const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-	const accessTokens = ctx.headers.authorization;
-	const client = getClient(BASE_URL, accessTokens);
-	try {
-		const multipartData = await ctx.file;
-		if (!multipartData) {
-			ctx.body = { error: "No image" };
-			ctx.status = 401;
-			return;
-		}
-		const data = await client.uploadMedia(multipartData);
-		ctx.body = convertAttachment(data.data as Entity.Attachment);
-	} catch (e: any) {
-		apiLogger.error(inspect(e));
-		ctx.status = 401;
-		ctx.body = e.response.data;
-	}
-});
-mastoFileRouter.post("/v2/media", upload.single("file"), async (ctx) => {
-	const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-	const accessTokens = ctx.headers.authorization;
-	const client = getClient(BASE_URL, accessTokens);
-	try {
-		const multipartData = await ctx.file;
-		if (!multipartData) {
-			ctx.body = { error: "No image" };
-			ctx.status = 401;
-			return;
-		}
-		const data = await client.uploadMedia(multipartData, ctx.request.body);
-		ctx.body = convertAttachment(data.data as Entity.Attachment);
-	} catch (e: any) {
-		apiLogger.error(inspect(e));
-		ctx.status = 401;
-		ctx.body = e.response.data;
-	}
-});
-
-mastoRouter.use(async (ctx, next) => {
-	if (ctx.request.query) {
-		if (!ctx.request.body || Object.keys(ctx.request.body).length === 0) {
-			ctx.request.body = ctx.request.query;
-		} else {
-			ctx.request.body = { ...ctx.request.body, ...ctx.request.query };
-		}
-	}
-	await next();
-});
-
-apiMastodonCompatible(mastoRouter);
+setupMastodonApi(mastoRouter);
 
 /**
  * Register endpoint handlers

packages/backend/src/server/api/mastodon/ApiMastodonCompatibleService.ts

@@ -1,163 +0,0 @@
-import Router from "@koa/router";
-import megalodon, { MegalodonInterface } from "megalodon";
-import { apiAuthMastodon } from "./endpoints/auth.js";
-import { apiAccountMastodon } from "./endpoints/account.js";
-import { apiStatusMastodon } from "./endpoints/status.js";
-import { apiFilterMastodon } from "./endpoints/filter.js";
-import { apiTimelineMastodon } from "./endpoints/timeline.js";
-import { apiNotificationsMastodon } from "./endpoints/notifications.js";
-import { apiSearchMastodon } from "./endpoints/search.js";
-import { getInstance } from "./endpoints/meta.js";
-import {
-	convertAccount,
-	convertAnnouncement,
-	convertFilter,
-} from "./converters.js";
-import { fromMastodonId } from "backend-rs";
-import { Users } from "@/models/index.js";
-import { IsNull } from "typeorm";
-import { apiLogger } from "../logger.js";
-import { inspect } from "node:util";
-
-export function getClient(
-	BASE_URL: string,
-	authorization: string | undefined,
-): MegalodonInterface {
-	const accessTokenArr = authorization?.split(" ") ?? [null];
-	const accessToken = accessTokenArr[accessTokenArr.length - 1];
-	const generator = (megalodon as any).default;
-	const client = generator(BASE_URL, accessToken) as MegalodonInterface;
-	return client;
-}
-
-export function apiMastodonCompatible(router: Router): void {
-	apiAuthMastodon(router);
-	apiAccountMastodon(router);
-	apiStatusMastodon(router);
-	apiFilterMastodon(router);
-	apiTimelineMastodon(router);
-	apiNotificationsMastodon(router);
-	apiSearchMastodon(router);
-
-	router.get("/v1/custom_emojis", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getInstanceCustomEmojis();
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.get("/v1/instance", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens); // we are using this here, because in private mode some info isnt
-		// displayed without being logged in
-		try {
-			const data = await client.getInstance();
-			const admin = await Users.findOne({
-				where: {
-					host: IsNull(),
-					isAdmin: true,
-					isDeleted: false,
-					isSuspended: false,
-				},
-				order: { id: "ASC" },
-			});
-			const contact =
-				admin == null
-					? null
-					: convertAccount((await client.getAccount(admin.id)).data);
-			ctx.body = await getInstance(data.data, contact);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.get("/v1/announcements", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getInstanceAnnouncements();
-			ctx.body = data.data.map((announcement) =>
-				convertAnnouncement(announcement),
-			);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.post<{ Params: { id: string } }>(
-		"/v1/announcements/:id/dismiss",
-		async (ctx) => {
-			const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-			const accessTokens = ctx.request.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.dismissInstanceAnnouncement(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
-		},
-	);
-
-	router.get("/v1/filters", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens); // we are using this here, because in private mode some info isnt
-		// displayed without being logged in
-		try {
-			const data = await client.getFilters();
-			ctx.body = data.data.map((filter) => convertFilter(filter));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.get("/v1/trends", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens); // we are using this here, because in private mode some info isnt
-		// displayed without being logged in
-		try {
-			const data = await client.getInstanceTrends();
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.get("/v1/preferences", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens); // we are using this here, because in private mode some info isnt
-		// displayed without being logged in
-		try {
-			const data = await client.getPreferences();
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-}

packages/backend/src/server/api/mastodon/converters.ts

@@ -1,95 +0,0 @@
-import type { Entity } from "megalodon";
-import { toMastodonId } from "backend-rs";
-
-function simpleConvert(data: any) {
-	// copy the object to bypass weird pass by reference bugs
-	const result = Object.assign({}, data);
-	result.id = toMastodonId(data.id);
-	return result;
-}
-
-export function convertAccount(account: Entity.Account) {
-	return simpleConvert(account);
-}
-export function convertAnnouncement(announcement: Entity.Announcement) {
-	return simpleConvert(announcement);
-}
-export function convertAttachment(attachment: Entity.Attachment) {
-	const converted = simpleConvert(attachment);
-	// ref: https://github.com/whitescent/Mastify/pull/102
-	if (converted.meta == null) return converted;
-	const result = {
-		...converted,
-		meta: {
-			...converted.meta,
-			original: {
-				...converted.meta,
-			},
-		},
-	};
-	return result;
-}
-export function convertFilter(filter: Entity.Filter) {
-	return simpleConvert(filter);
-}
-export function convertList(list: Entity.List) {
-	return simpleConvert(list);
-}
-export function convertFeaturedTag(tag: Entity.FeaturedTag) {
-	return simpleConvert(tag);
-}
-
-export function convertNotification(notification: Entity.Notification) {
-	notification.account = convertAccount(notification.account);
-	notification.id = toMastodonId(notification.id);
-	if (notification.status)
-		notification.status = convertStatus(notification.status);
-	if (notification.reaction)
-		notification.reaction = convertReaction(notification.reaction);
-	return notification;
-}
-
-export function convertPoll(poll: Entity.Poll) {
-	return simpleConvert(poll);
-}
-export function convertReaction(reaction: Entity.Reaction) {
-	if (reaction.accounts) {
-		reaction.accounts = reaction.accounts.map(convertAccount);
-	}
-	return reaction;
-}
-export function convertRelationship(relationship: Entity.Relationship) {
-	return simpleConvert(relationship);
-}
-
-export function convertStatus(status: Entity.Status) {
-	status.account = convertAccount(status.account);
-	status.id = toMastodonId(status.id);
-	if (status.in_reply_to_account_id)
-		status.in_reply_to_account_id = toMastodonId(status.in_reply_to_account_id);
-	if (status.in_reply_to_id)
-		status.in_reply_to_id = toMastodonId(status.in_reply_to_id);
-	status.media_attachments = status.media_attachments.map((attachment) =>
-		convertAttachment(attachment),
-	);
-	status.mentions = status.mentions.map((mention) => ({
-		...mention,
-		id: toMastodonId(mention.id),
-	}));
-	if (status.poll) status.poll = convertPoll(status.poll);
-	if (status.reblog) status.reblog = convertStatus(status.reblog);
-	if (status.quote) status.quote = convertStatus(status.quote);
-	status.reactions = status.reactions.map(convertReaction);
-
-	return status;
-}
-
-export function convertConversation(conversation: Entity.Conversation) {
-	conversation.id = toMastodonId(conversation.id);
-	conversation.accounts = conversation.accounts.map(convertAccount);
-	if (conversation.last_status) {
-		conversation.last_status = convertStatus(conversation.last_status);
-	}
-
-	return conversation;
-}

packages/backend/src/server/api/mastodon/converters/announcement.ts

@@ -0,0 +1,55 @@
+import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js";
+import mfm from "mfm-js";
+import type { Announcement } from "@/models/entities/announcement.js";
+import type { MastoContext } from "..";
+import { I18n } from "@/misc/i18n.js";
+import locales from "../../../../../../../locales/index.mjs";
+
+export class AnnouncementConverter {
+	public static async encode(
+		announcement: Announcement,
+		isRead: boolean,
+		lang = "en-US",
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Announcement> {
+		const locale = locales[lang] || locales["en-US"];
+		const i18n = new I18n(locale);
+
+		return {
+			id: announcement.id,
+			content: `<h1>${
+				(await MfmHelpers.toHtml(
+					mfm.parse(announcement.title),
+					[],
+					null,
+					false,
+					null,
+					ctx,
+				)) ?? i18n.t("announcement")
+			}</h1>${
+				(await MfmHelpers.toHtml(
+					mfm.parse(announcement.text),
+					[],
+					null,
+					false,
+					null,
+					ctx,
+				)) ?? ""
+			}`,
+			starts_at: null,
+			ends_at: null,
+			published: true,
+			all_day: false,
+			published_at: announcement.createdAt.toISOString(),
+			updated_at:
+				announcement.updatedAt?.toISOString() ??
+				announcement.createdAt.toISOString(),
+			read: isRead,
+			mentions: [], //FIXME
+			statuses: [],
+			tags: [],
+			emojis: [], //FIXME
+			reactions: [],
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/emoji.ts

@@ -0,0 +1,13 @@
+import type { PopulatedEmoji } from "@/misc/populate-emojis.js";
+
+export class EmojiConverter {
+	public static encode(e: PopulatedEmoji): MastodonEntity.Emoji {
+		return {
+			shortcode: e.name,
+			static_url: e.url,
+			url: e.url,
+			visible_in_picker: true,
+			category: undefined,
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/file.ts

@@ -0,0 +1,54 @@
+import type { Packed } from "@/misc/schema.js";
+import type { MastoContext } from "..";
+
+export class FileConverter {
+	public static encode(
+		f: Packed<"DriveFile">,
+		ctx?: MastoContext,
+	): MastodonEntity.Attachment {
+		return {
+			id: f.id,
+			type: this.encodefileType(f.type),
+			url: f.url ?? "",
+			remote_url: f.url,
+			preview_url: f.thumbnailUrl ?? null,
+			text_url: f.url,
+			meta: {
+				width: f.properties.width,
+				height: f.properties.height,
+				original: {
+					width: f.properties.width,
+					height: f.properties.height,
+					size:
+						f.properties.width && f.properties.height
+							? `${f.properties.width}x${f.properties.height}`
+							: undefined,
+					aspect:
+						f.properties.width && f.properties.height
+							? f.properties.width / f.properties.height
+							: undefined,
+				},
+			},
+			description: f.comment,
+			blurhash: f.blurhash,
+		};
+	}
+
+	private static encodefileType(
+		s: string,
+	): "unknown" | "image" | "gifv" | "video" | "audio" {
+		if (s === "image/gif") {
+			return "gifv";
+		}
+		if (s.includes("image")) {
+			return "image";
+		}
+		if (s.includes("video")) {
+			return "video";
+		}
+		if (s.includes("audio")) {
+			return "audio";
+		}
+		return "unknown";
+	}
+}

packages/backend/src/server/api/mastodon/converters/mention.ts

@@ -0,0 +1,28 @@
+import type { User } from "@/models/entities/user.js";
+import { config } from "@/config.js";
+import type { IMentionedRemoteUsers } from "@/models/entities/note.js";
+
+export class MentionConverter {
+	public static encode(
+		u: User,
+		m: IMentionedRemoteUsers,
+	): MastodonEntity.Mention {
+		let acct = u.username;
+		let acctUrl = `https://${u.host || config.host}/@${u.username}`;
+		let url: string | null = null;
+		if (u.host) {
+			const info = m.find(
+				(r) => r.username === u.username && r.host === u.host,
+			);
+			acct = `${u.username}@${u.host}`;
+			acctUrl = `https://${u.host}/@${u.username}`;
+			if (info) url = info.url ?? info.uri;
+		}
+		return {
+			id: u.id,
+			username: u.username,
+			acct: acct,
+			url: url ?? acctUrl,
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/mfm.ts

@@ -0,0 +1,9 @@
+export const escapeMFM = (text: string): string =>
+	text
+		.replace(/&/g, "&")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&#39;")
+		.replace(/`/g, "&#x60;")
+		.replace(/\r?\n/g, "<br>");

packages/backend/src/server/api/mastodon/converters/note.ts

@@ -0,0 +1,647 @@
+import type { ILocalUser, User } from "@/models/entities/user.js";
+import { getNote } from "@/server/api/common/getters.js";
+import type { Note } from "@/models/entities/note.js";
+import { config } from "@/config.js";
+import mfm, { type MfmLink, type MfmUrl } from "mfm-js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { VisibilityConverter } from "@/server/api/mastodon/converters/visibility.js";
+import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
+import {
+	aggregateNoteEmojis,
+	type PopulatedEmoji,
+	populateEmojis,
+	prefetchEmojis,
+} from "@/misc/populate-emojis.js";
+import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js";
+import {
+	DriveFiles,
+	NoteFavorites,
+	NoteReactions,
+	Notes,
+	NoteThreadMutings,
+	UserNotePinings,
+} from "@/models/index.js";
+import { decodeReaction } from "backend-rs";
+import { MentionConverter } from "@/server/api/mastodon/converters/mention.js";
+import { PollConverter } from "@/server/api/mastodon/converters/poll.js";
+import { populatePoll } from "@/models/repositories/note.js";
+import { FileConverter } from "@/server/api/mastodon/converters/file.js";
+import { awaitAll } from "@/prelude/await-all.js";
+import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
+import { In, IsNull } from "typeorm";
+import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js";
+import {
+	getStubMastoContext,
+	type MastoContext,
+} from "@/server/api/mastodon/index.js";
+import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
+import isQuote from "@/misc/is-quote.js";
+import { unique } from "@/prelude/array.js";
+import type { NoteReaction } from "@/models/entities/note-reaction.js";
+import { Cache } from "@/misc/cache.js";
+import { isFiltered } from "@/misc/is-filtered.js";
+import { unfurl } from "unfurl.js";
+import type { ScheduledNote } from "@/models/entities/scheduled-note";
+
+export class NoteConverter {
+	private static noteContentHtmlCache = new Cache<string | null>(
+		"html:note:content",
+		60 * 60,
+	);
+	private static cardCache = new Cache<MastodonEntity.Card | null>(
+		"note:card",
+		60 * 60,
+	);
+	public static async encode(
+		note: Note,
+		ctx: MastoContext,
+		recurseCounter = 2,
+	): Promise<MastodonEntity.Status> {
+		const user = ctx.user as ILocalUser | null;
+		const noteUser = note.user ?? UserHelpers.getUserCached(note.userId, ctx);
+
+		if (!(await Notes.isVisibleForMe(note, user?.id ?? null)))
+			throw new Error("Cannot encode note not visible for user");
+
+		const host = Promise.resolve(noteUser).then(
+			(noteUser) => noteUser.host ?? null,
+		);
+
+		const reactionEmojiNames = Object.keys(note.reactions)
+			.filter((x) => x?.startsWith(":"))
+			.map((x) => decodeReaction(x).reaction)
+			.map((x) => x.replace(/:/g, ""));
+
+		const populated = host.then(async (host) =>
+			populateEmojis(note.emojis.concat(reactionEmojiNames), host),
+		);
+
+		const noteEmoji = populated.then((noteEmoji) =>
+			noteEmoji
+				.filter((e) => e.name.indexOf("@") === -1)
+				.map((e) => EmojiConverter.encode(e)),
+		);
+
+		const reactionCount = Object.values(note.reactions).reduce(
+			(a, b) => a + b,
+			0,
+		);
+
+		const aggregateReaction = (
+			ctx.reactionAggregate as Map<string, NoteReaction | null>
+		)?.get(note.id);
+
+		const reaction =
+			aggregateReaction !== undefined
+				? aggregateReaction
+				: user
+					? NoteReactions.findOneBy({
+							userId: user.id,
+							noteId: note.id,
+						})
+					: null;
+
+		const isFavorited = Promise.resolve(reaction).then((p) => !!p);
+
+		const isReblogged =
+			(ctx.renoteAggregate as Map<string, boolean>)?.get(note.id) ??
+			(user
+				? Notes.exists({
+						where: {
+							userId: user.id,
+							renoteId: note.id,
+							text: IsNull(),
+						},
+					})
+				: null);
+
+		const renote =
+			note.renote ??
+			(note.renoteId && recurseCounter > 0
+				? getNote(note.renoteId, user)
+				: null);
+
+		const isBookmarked =
+			(ctx.bookmarkAggregate as Map<string, boolean>)?.get(note.id) ??
+			(user
+				? NoteFavorites.exists({
+						where: {
+							userId: user.id,
+							noteId: note.id,
+						},
+						take: 1,
+					})
+				: false);
+
+		const isMuted =
+			(ctx.mutingAggregate as Map<string, boolean>)?.get(
+				note.threadId ?? note.id,
+			) ??
+			(user
+				? NoteThreadMutings.exists({
+						where: {
+							userId: user.id,
+							threadId: note.threadId || note.id,
+						},
+					})
+				: false);
+
+		const files = DriveFiles.packMany(note.fileIds);
+
+		const mentions = Promise.all(
+			note.mentions.map((p) =>
+				UserHelpers.getUserCached(p, ctx)
+					.then((u) =>
+						MentionConverter.encode(u, JSON.parse(note.mentionedRemoteUsers)),
+					)
+					.catch(() => null),
+			),
+		).then((p) => p.filter((m) => m)) as Promise<MastodonEntity.Mention[]>;
+
+		const quoteUri = Promise.resolve(renote).then((renote) => {
+			if (!renote || !isQuote(note)) return null;
+			return renote.url ?? renote.uri ?? `${config.url}/notes/${renote.id}`;
+		});
+
+		const identifier = `${note.id}:${(
+			note.updatedAt ?? note.createdAt
+		).getTime()}`;
+
+		const text = quoteUri.then((quoteUri) =>
+			note.text !== null
+				? quoteUri !== null
+					? note.text
+							.replaceAll(`RE: ${quoteUri}`, "")
+							.replaceAll(quoteUri, "")
+							.trimEnd()
+					: note.text
+				: null,
+		);
+
+		const content = this.noteContentHtmlCache
+			.fetch(
+				identifier,
+				async () =>
+					text.then((text) =>
+						text !== null
+							? quoteUri
+									.then((quoteUri) =>
+										MfmHelpers.toHtml(
+											mfm.parse(text),
+											JSON.parse(note.mentionedRemoteUsers),
+											note.userHost,
+											false,
+											quoteUri,
+											ctx,
+										),
+									)
+									.then((p) => p ?? escapeMFM(text))
+							: "",
+					),
+				true,
+			)
+			.then((p) => p ?? "");
+
+		const card = text
+			.then(async (text) => this.extractUrlFromMfm(text))
+			.then(async (urls) =>
+				!urls
+					? null
+					: Promise.race([
+							this.cardCache.fetch(
+								identifier,
+								async () => this.generateCard(urls, note.lang ?? undefined),
+								true,
+							),
+							new Promise<null>((resolve) =>
+								setTimeout(() => resolve(null), 5000),
+							), // Timeout card generation after 5 seconds
+						]),
+			);
+
+		const isPinned =
+			(ctx.pinAggregate as Map<string, boolean>)?.get(note.id) ??
+			(user && note.userId === user.id
+				? UserNotePinings.exists({
+						where: { userId: user.id, noteId: note.id },
+					})
+				: undefined);
+
+		const tags = note.tags.map((tag) => {
+			return {
+				name: tag,
+				url: `${config.url}/tags/${tag}`,
+			} as MastodonEntity.Tag;
+		});
+
+		const reblog = Promise.resolve(renote).then((renote) =>
+			recurseCounter > 0 && renote
+				? this.encode(
+						renote,
+						ctx,
+						isQuote(renote) && !isQuote(note) ? --recurseCounter : 0,
+					)
+				: null,
+		);
+
+		const filtered = isFiltered(note, user).then((res) => {
+			if (
+				!res ||
+				ctx.filterContext == null ||
+				!["home", "public"].includes(ctx.filterContext)
+			)
+				return null;
+			return [
+				{
+					filter: {
+						id: "0",
+						title: "Hard word mutes",
+						context: ["home", "public"],
+						expires_at: null,
+						filter_action: "hide",
+						keywords: [],
+						statuses: [],
+					},
+				} as MastodonEntity.FilterResult,
+			];
+		});
+
+		return await awaitAll({
+			id: note.id,
+			uri: note.uri ?? `https://${config.host}/notes/${note.id}`,
+			url: note.url ?? note.uri ?? `https://${config.host}/notes/${note.id}`,
+			account: Promise.resolve(noteUser).then((p) =>
+				UserConverter.encode(p, ctx),
+			),
+			in_reply_to_id: note.replyId,
+			in_reply_to_account_id: note.replyUserId,
+			reblog: reblog.then((reblog) => (!isQuote(note) ? reblog : null)),
+			content: content,
+			content_type: "text/x.misskeymarkdown",
+			text: text,
+			created_at: note.createdAt.toISOString(),
+			emojis: noteEmoji,
+			replies_count: reblog.then(
+				(reblog) =>
+					(!isQuote(note) ? reblog?.replies_count : note.repliesCount) ?? 0,
+			),
+			reblogs_count: reblog.then(
+				(reblog) =>
+					(!isQuote(note) ? reblog?.reblogs_count : note.renoteCount) ?? 0,
+			),
+			favourites_count: reactionCount,
+			reblogged:
+				isReblogged || reblog.then((reblog) => !!(reblog && !isQuote(note))),
+			favourited: isFavorited,
+			muted: isMuted,
+			sensitive: files.then((files) =>
+				files.length > 0 ? files.some((f) => f.isSensitive) : false,
+			),
+			spoiler_text: note.cw ? note.cw : "",
+			visibility: VisibilityConverter.encode(note.visibility),
+			media_attachments: files.then((files) =>
+				files.length > 0 ? files.map((f) => FileConverter.encode(f)) : [],
+			),
+			mentions: mentions,
+			tags: tags,
+			card: card,
+			poll: note.hasPoll
+				? populatePoll(note, user?.id ?? null).then((p) =>
+						noteEmoji.then((emojis) =>
+							PollConverter.encode(p, note.id, emojis),
+						),
+					)
+				: null,
+			application: null, //FIXME
+			language: note.lang,
+			pinned: isPinned,
+			reactions: populated.then((populated) =>
+				Promise.resolve(reaction).then((reaction) =>
+					this.encodeReactions(
+						note.reactions,
+						reaction?.reaction,
+						populated,
+						ctx,
+					),
+				),
+			),
+			bookmarked: isBookmarked,
+			quote: reblog.then((reblog) => (isQuote(note) ? reblog : null)),
+			edited_at: note.updatedAt?.toISOString() ?? null,
+			filtered: filtered,
+		});
+	}
+
+	public static async encodeMany(
+		notes: Note[],
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Status[]> {
+		await this.aggregateData(notes, ctx);
+		const encoded = notes.map((n) => this.encode(n, ctx));
+		return Promise.all(encoded);
+	}
+
+	public static async aggregateData(
+		notes: Note[],
+		ctx: MastoContext,
+	): Promise<void> {
+		if (notes.length === 0) return;
+
+		const user = ctx.user as ILocalUser | null;
+		const reactionAggregate = new Map<Note["id"], NoteReaction | null>();
+		const renoteAggregate = new Map<Note["id"], boolean>();
+		const mutingAggregate = new Map<Note["id"], boolean>();
+		const bookmarkAggregate = new Map<Note["id"], boolean>();
+		const pinAggregate = new Map<Note["id"], boolean>();
+
+		const renoteIds = notes
+			.map((n) => n.renoteId)
+			.filter((n): n is string => n != null);
+
+		const noteIds = unique(notes.map((n) => n.id));
+		const targets = unique([...noteIds, ...renoteIds]);
+
+		if (user?.id != null) {
+			const mutingTargets = unique([...notes.map((n) => n.threadId ?? n.id)]);
+			const pinTargets = unique([
+				...notes.filter((n) => n.userId === user.id).map((n) => n.id),
+			]);
+
+			const reactions = await NoteReactions.findBy({
+				userId: user.id,
+				noteId: In(targets),
+			});
+
+			const renotes = await Notes.createQueryBuilder("note")
+				.select("note.renoteId")
+				.where("note.userId = :meId", { meId: user.id })
+				.andWhere("note.renoteId IN (:...targets)", { targets })
+				.andWhere("note.text IS NULL")
+				.andWhere("note.hasPoll = FALSE")
+				.andWhere(`note.fileIds = '{}'`)
+				.getMany();
+
+			const mutings = await NoteThreadMutings.createQueryBuilder("muting")
+				.select("muting.threadId")
+				.where("muting.userId = :meId", { meId: user.id })
+				.andWhere("muting.threadId IN (:...targets)", {
+					targets: mutingTargets,
+				})
+				.getMany();
+
+			const bookmarks = await NoteFavorites.createQueryBuilder("bookmark")
+				.select("bookmark.noteId")
+				.where("bookmark.userId = :meId", { meId: user.id })
+				.andWhere("bookmark.noteId IN (:...targets)", { targets })
+				.getMany();
+
+			const pins =
+				pinTargets.length > 0
+					? await UserNotePinings.createQueryBuilder("pin")
+							.select("pin.noteId")
+							.where("pin.userId = :meId", { meId: user.id })
+							.andWhere("pin.noteId IN (:...targets)", { targets: pinTargets })
+							.getMany()
+					: [];
+
+			for (const target of targets) {
+				reactionAggregate.set(
+					target,
+					reactions.find((r) => r.noteId === target) ?? null,
+				);
+				renoteAggregate.set(
+					target,
+					!!renotes.find((n) => n.renoteId === target),
+				);
+				bookmarkAggregate.set(
+					target,
+					!!bookmarks.find((b) => b.noteId === target),
+				);
+			}
+
+			for (const target of mutingTargets) {
+				mutingAggregate.set(
+					target,
+					!!mutings.find((m) => m.threadId === target),
+				);
+			}
+
+			for (const target of pinTargets) {
+				mutingAggregate.set(target, !!pins.find((m) => m.noteId === target));
+			}
+		}
+
+		ctx.reactionAggregate = reactionAggregate;
+		ctx.renoteAggregate = renoteAggregate;
+		ctx.mutingAggregate = mutingAggregate;
+		ctx.bookmarkAggregate = bookmarkAggregate;
+		ctx.pinAggregate = pinAggregate;
+
+		const users = notes.filter((p) => !!p.user).map((p) => p.user as User);
+		const renoteUserIds = notes
+			.filter((p) => p.renoteUserId !== null)
+			.map((p) => p.renoteUserId as string);
+		await UserConverter.aggregateData([...users], ctx);
+		await prefetchEmojis(aggregateNoteEmojis(notes));
+	}
+
+	private static encodeReactions(
+		reactions: Record<string, number>,
+		myReaction: string | undefined,
+		populated: PopulatedEmoji[],
+		ctx: MastoContext,
+	): MastodonEntity.Reaction[] {
+		// Client compatibility: SoraSNS requires `reactions` to be a `Dictionary<string, int>`.
+		if (ctx?.tokenApp?.name === "SoraSNS for iPad") {
+			return reactions as unknown as MastodonEntity.Reaction[];
+		}
+		return Object.keys(reactions)
+			.map((key) => {
+				const isCustom = key.startsWith(":") && key.endsWith(":");
+				const name = isCustom ? key.substring(1, key.length - 1) : key;
+				const populatedName =
+					isCustom && name.indexOf("@") === -1 ? `${name}@.` : name;
+				const url = isCustom
+					? populated.find((p) => p.name === populatedName)?.url
+					: undefined;
+
+				return {
+					count: reactions[key],
+					me: key === myReaction,
+					name: name,
+					url: url,
+					static_url: url,
+				};
+			})
+			.filter((r) => r.count > 0);
+	}
+
+	public static async encodeEvent(
+		note: Note,
+		user: ILocalUser | undefined,
+		filterContext?: string,
+	): Promise<MastodonEntity.Status> {
+		const ctx = getStubMastoContext(user, filterContext);
+		NoteHelpers.fixupEventNote(note);
+		return NoteConverter.encode(note, ctx);
+	}
+
+	private static removeHash(x: string) {
+		return x.replace(/#[^#]*$/, "");
+	}
+
+	private static extractUrlFromMfm(text: string | null): string[] {
+		if (!text) return [];
+		const nodes = mfm.parse(text);
+		const urlNodes = mfm.extract(nodes, (node) => {
+			return (
+				node.type === "url" || (node.type === "link" && !node.props.silent)
+			);
+		}) as (MfmUrl | MfmLink)[];
+		const urls: string[] = unique(urlNodes.map((x) => x.props.url));
+
+		return urls.reduce((array, url) => {
+			const urlWithoutHash = this.removeHash(url);
+			if (!array.map((x) => this.removeHash(x)).includes(urlWithoutHash))
+				array.push(url);
+			return array;
+		}, [] as string[]);
+	}
+
+	/** Generate URL preview metadata from the first possible URL in the list provided. */
+	private static async generateCard(
+		urls: string[],
+		lang?: string,
+	): Promise<MastodonEntity.Card | null> {
+		if (urls.length === 0) return null;
+		for (const url of urls) {
+			try {
+				const summary = await unfurl(url, {
+					oembed: true,
+					follow: 10,
+					compress: true,
+					headers: { "Accept-Language": lang ?? "en-US" },
+				});
+				if (summary) {
+					return {
+						url: summary.canonical_url ?? url,
+						title: summary.title ?? "",
+						description: summary.description ?? "",
+						image:
+							summary.oEmbed?.thumbnails?.[0]?.url ??
+							summary.open_graph?.images?.[0]?.secure_url ??
+							summary.open_graph?.images?.[0]?.url ??
+							summary.twitter_card?.images?.[0]?.url ??
+							null,
+						type:
+							summary.oEmbed?.type ??
+							(summary.open_graph?.videos ||
+							summary.open_graph?.audio ||
+							summary.twitter_card?.players
+								? "video"
+								: "link"),
+						author_name:
+							summary.author ??
+							summary.oEmbed?.author_name ??
+							summary.open_graph?.article?.author ??
+							summary.twitter_card?.creator ??
+							"",
+						author_url: summary.oEmbed?.author_name ?? "",
+						provider_name:
+							summary.oEmbed?.provider_name ??
+							summary.open_graph?.site_name ??
+							summary.twitter_card?.site ??
+							"",
+						provider_url: summary.oEmbed?.provider_url ?? "",
+						html: (summary.oEmbed as { html?: string })?.html ?? "",
+						width:
+							summary.oEmbed?.thumbnails?.[0]?.width ??
+							summary.open_graph?.images?.[0]?.width ??
+							summary.open_graph?.videos?.[0]?.width ??
+							0,
+						height:
+							summary.oEmbed?.thumbnails?.[0]?.height ??
+							summary.open_graph?.images?.[0]?.height ??
+							summary.open_graph?.videos?.[0]?.height ??
+							0,
+						embed_url:
+							summary.open_graph?.videos?.[0]?.stream ??
+							summary.open_graph?.videos?.[0]?.url ??
+							"",
+						blurhash: null,
+						published_at: summary.open_graph?.article?.published_time ?? null,
+						image_description: summary.open_graph?.images?.[0]?.alt ?? "",
+						language: lang ?? null,
+					};
+				}
+			} catch {
+				// no op.
+			}
+		}
+		return null;
+	}
+
+	/** Encode a schduled note. */
+	public static async encodeScheduledNote(
+		scheduledNote: ScheduledNote,
+		ctx: MastoContext,
+	): Promise<MastodonEntity.ScheduledStatus> {
+		const { note, user } = scheduledNote;
+
+		const renote =
+			note.renote ?? (note.renoteId ? getNote(note.renoteId, user) : null);
+		const quoteUri = Promise.resolve(renote).then((renote) => {
+			if (!renote || !isQuote(note)) return null;
+			return renote.url ?? renote.uri ?? `${config.url}/notes/${renote.id}`;
+		});
+		const text = quoteUri.then((quoteUri) =>
+			note.text !== null
+				? quoteUri !== null
+					? note.text
+							.replaceAll(`RE: ${quoteUri}`, "")
+							.replaceAll(quoteUri, "")
+							.trimEnd()
+					: note.text
+				: "",
+		);
+
+		const files = DriveFiles.packMany(note.fileIds);
+
+		const a = await awaitAll({
+			id: scheduledNote.noteId,
+			scheduled_at: scheduledNote.scheduledAt.toISOString(),
+			params: {
+				text,
+				poll: note.hasPoll
+					? populatePoll(note, user?.id ?? null).then((p) =>
+							PollConverter.encodeScheduledPoll(p),
+						)
+					: null,
+				media_ids: note.fileIds,
+				sensitive: files.then((files) =>
+					files.length > 0 ? files.some((f) => f.isSensitive) : false,
+				),
+				spoiler_text: note.cw || "",
+				visibility: VisibilityConverter.encode(note.visibility),
+				in_reply_to_id: note.replyId,
+				language: note.lang,
+				application_id: 0,
+				idempotency: scheduledNote.id,
+				with_rate_limit: false,
+			},
+			media_attachments: files.then((files) =>
+				files.length > 0 ? files.map((f) => FileConverter.encode(f)) : [],
+			),
+		});
+		return a;
+	}
+
+	/** Encode an array of schduled notes. */
+	public static async encodeManyScheduledNotes(
+		scheduledNotes: ScheduledNote[],
+		ctx: MastoContext,
+	): Promise<MastodonEntity.ScheduledStatus[]> {
+		const encoded = scheduledNotes.map((n) => this.encodeScheduledNote(n, ctx));
+		return Promise.all(encoded);
+	}
+}

packages/backend/src/server/api/mastodon/converters/notification.ts

@@ -0,0 +1,278 @@
+import type { ILocalUser, User } from "@/models/entities/user.js";
+import type { Notification } from "@/models/entities/notification.js";
+import type { notificationTypes } from "@/types.js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
+import { awaitAll } from "@/prelude/await-all.js";
+import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
+import { getNote } from "@/server/api/common/getters.js";
+import {
+	getStubMastoContext,
+	type MastoContext,
+} from "@/server/api/mastodon/index.js";
+import { Notifications } from "@/models/index.js";
+import isQuote from "@/misc/is-quote.js";
+import { unique } from "@/prelude/array.js";
+import type { Note } from "@/models/entities/note.js";
+import type { SwSubscription } from "@/models/entities/sw-subscription.js";
+import { fetchMeta } from "backend-rs";
+import { getNoteSummary } from "backend-rs";
+import type { Packed } from "@/misc/schema";
+import { I18n } from "@/misc/i18n.js";
+import locales from "../../../../../../../locales/index.mjs";
+
+type NotificationType = (typeof notificationTypes)[number];
+
+export class NotificationConverter {
+	public static async encode(
+		notification: Notification,
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Notification> {
+		const localUser = ctx.user as ILocalUser;
+		if (notification.notifieeId !== localUser.id)
+			throw new Error("User is not recipient of notification");
+
+		const account = notification.notifierId
+			? UserHelpers.getUserCached(notification.notifierId, ctx).then((p) =>
+					UserConverter.encode(p, ctx),
+				)
+			: UserConverter.encode(localUser, ctx);
+
+		let result = {
+			id: notification.id,
+			account: account,
+			created_at: notification.createdAt.toISOString(),
+			type: this.encodeNotificationType(notification.type),
+		};
+
+		const note =
+			notification.note ??
+			(notification.noteId
+				? await getNote(notification.noteId, localUser)
+				: null);
+
+		if (note) {
+			const encodedNote =
+				note.renoteId !== null && !isQuote(note)
+					? getNote(note.renoteId, localUser).then((note) =>
+							NoteConverter.encode(note, ctx),
+						)
+					: NoteConverter.encode(note, ctx);
+			result = Object.assign(result, {
+				status: encodedNote,
+			});
+			if (result.type === "poll") {
+				result = Object.assign(result, {
+					account: encodedNote.then((p) => p.account),
+				});
+			}
+			if (notification.reaction) {
+				// FIXME: Implement reactions;
+			}
+		}
+		return awaitAll(result);
+	}
+
+	public static async encodeMany(
+		notifications: Notification[],
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Notification[]> {
+		await this.aggregateData(notifications, ctx);
+		const encoded = notifications.map((u) => this.encode(u, ctx));
+		return Promise.all(encoded).then(
+			(p) => p.filter((n) => n !== null) as MastodonEntity.Notification[],
+		);
+	}
+
+	private static async aggregateData(
+		notifications: Notification[],
+		ctx: MastoContext,
+	): Promise<void> {
+		if (notifications.length === 0) return;
+		const notes = unique(
+			notifications.filter((p) => p.note != null).map((n) => n.note as Note),
+		);
+		const users = unique(
+			notifications
+				.filter((p) => p.notifier != null)
+				.map((n) => n.notifier as User)
+				.concat(
+					notifications
+						.filter((p) => p.notifiee != null)
+						.map((n) => n.notifiee as User),
+				),
+		);
+		await NoteConverter.aggregateData(notes, ctx);
+		await UserConverter.aggregateData(users, ctx);
+	}
+
+	private static encodeNotificationType(
+		t: NotificationType,
+	): MastodonEntity.NotificationType {
+		// FIXME: Implement custom notification for followRequestAccepted
+		// FIXME: Implement mastodon notification type 'update' on misskey side
+		switch (t) {
+			case "follow":
+				return "follow";
+			case "mention":
+			case "reply":
+				return "mention";
+			case "renote":
+				return "reblog";
+			case "quote":
+				return "reblog";
+			case "reaction":
+				return "favourite";
+			case "pollEnded":
+				return "poll";
+			case "receiveFollowRequest":
+				return "follow_request";
+			case "followRequestAccepted":
+			case "pollVote":
+			case "groupInvited":
+			case "app":
+				throw new Error(`Notification type ${t} not supported`);
+		}
+	}
+
+	public static async encodeNotificationTypeOrDefault(
+		t: NotificationType,
+	): Promise<string> {
+		try {
+			return this.encodeNotificationType(t);
+		} catch (e) {
+			return t;
+		}
+	}
+
+	public static async encodeEvent(
+		target: Notification["id"],
+		user: ILocalUser,
+		filterContext?: string,
+	): Promise<MastodonEntity.Notification | null> {
+		const ctx = getStubMastoContext(user, filterContext);
+		const notification = await Notifications.findOneByOrFail({ id: target });
+		return this.encode(notification, ctx).catch((_) => null);
+	}
+
+	public static async encodeSubscription(
+		subscription: SwSubscription,
+		ctx: MastoContext,
+	): Promise<MastodonEntity.PushSubscription> {
+		const instance = await fetchMeta(true);
+		return {
+			id: subscription.id,
+			endpoint: subscription.endpoint,
+			server_key: instance.swPublicKey ?? "",
+			alerts: {
+				follow: subscription.subscriptionTypes.includes("follow"),
+				favourite: subscription.subscriptionTypes.includes("favourite"),
+				mention: subscription.subscriptionTypes.includes("mention"),
+				reblog: subscription.subscriptionTypes.includes("reblog"),
+				poll: subscription.subscriptionTypes.includes("poll"),
+				status: subscription.subscriptionTypes.includes("status"),
+			},
+		};
+	}
+
+	public static async encodePushNotificationPayloadForRust(
+		body: Packed<"Notification">,
+		lang = "en-US",
+	): Promise<Partial<MastodonEntity.NotificationPayload>> {
+		const locale = locales[lang] || locales["en-US"];
+		const i18n = new I18n(locale);
+
+		let preferred_locale = lang;
+		let notification_id = "";
+		let notification_type = "others";
+		let icon: string | undefined = undefined;
+		let title = i18n.t("notificationType");
+		let description = "";
+
+		const notificationBody = body;
+		preferred_locale = notificationBody.note?.lang ?? preferred_locale;
+		notification_id = notificationBody.id;
+		notification_type = await this.encodeNotificationTypeOrDefault(
+			notificationBody.type,
+		);
+		const effectiveNote =
+			notificationBody.note?.renote ?? notificationBody.note;
+
+		icon =
+			notificationBody.user?.avatarUrl ??
+			notificationBody.note?.user.avatarUrl ??
+			notificationBody.icon ??
+			undefined;
+		const displayName =
+			notificationBody.user?.name ||
+			(notificationBody.user?.host &&
+				`@${notificationBody.user?.username}@${notificationBody.user?.host}`) ||
+			(notificationBody.user?.username &&
+				`@${notificationBody.user?.username}`) ||
+			"Someone";
+		const username =
+			(notificationBody.user?.host &&
+				`@${notificationBody.user?.username}@${notificationBody.user?.host}`) ||
+			(notificationBody.user?.username &&
+				`@${notificationBody.user?.username}`) ||
+			"";
+
+		// FIXME: all notification title i18n strings should take `name` as a parameter
+		switch (notificationBody.type) {
+			case "mention":
+				title = i18n.t("_notification.youGotMention", { name: displayName });
+				break;
+			case "reply":
+				title = i18n.t("_notification.youGotReply", { name: displayName });
+				break;
+			case "renote":
+				title = i18n.t("_notification.youRenoted", { name: displayName });
+				break;
+			case "quote":
+				title = i18n.t("_notification.youGotQuote", { name: displayName });
+				break;
+			case "reaction":
+				title = `${displayName} ${i18n.t("_notification.reacted")}`;
+				break;
+			case "pollVote":
+				title = i18n.t("_notification.youGotPoll", { name: displayName });
+				break;
+			case "pollEnded":
+				title = i18n.t("_notification.pollEnded");
+				break;
+			case "followRequestAccepted":
+				title = i18n.t("_notification.yourFollowRequestAccepted");
+				break;
+			case "groupInvited":
+				title = i18n.t("_notification.youWereInvitedToGroup", {
+					userName: displayName,
+				});
+				break;
+			case "follow":
+				title = `${displayName} ${i18n.t("_notification.youWereFollowed")}`;
+				break;
+			case "receiveFollowRequest":
+				title = i18n.t("_notification.youReceivedFollowRequest");
+				break;
+			case "app":
+				title = `${notificationBody.header}`;
+				break;
+			default:
+				title = `${i18n.t("notificationType")} ${notificationBody.type}`;
+		}
+		description =
+			(effectiveNote && getNoteSummary(effectiveNote)) ||
+			notificationBody.body ||
+			username ||
+			"";
+
+		return {
+			preferred_locale,
+			notification_id,
+			notification_type,
+			icon,
+			title,
+			body: description,
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/poll.ts

@@ -0,0 +1,58 @@
+type Choice = {
+	text: string;
+	votes: number;
+	isVoted: boolean;
+};
+
+type Poll = {
+	multiple: boolean;
+	expiresAt: Date | null;
+	choices: Array<Choice>;
+};
+
+export class PollConverter {
+	public static encode(
+		p: Poll,
+		noteId: string,
+		emojis: MastodonEntity.Emoji[],
+	): MastodonEntity.Poll {
+		const now = new Date();
+		const count = p.choices.reduce((sum, choice) => sum + choice.votes, 0);
+		return {
+			id: noteId,
+			expires_at: p.expiresAt?.toISOString() ?? null,
+			expired: p.expiresAt == null ? false : now > p.expiresAt,
+			multiple: p.multiple,
+			votes_count: count,
+			options: p.choices.map((c) => this.encodeChoice(c)),
+			emojis: emojis,
+			voted: p.choices.some((c) => c.isVoted),
+			own_votes: p.choices
+				.filter((c) => c.isVoted)
+				.map((c) => p.choices.indexOf(c)),
+		};
+	}
+
+	private static encodeChoice(c: Choice): MastodonEntity.PollOption {
+		return {
+			title: c.text,
+			votes_count: c.votes,
+		};
+	}
+
+	public static encodeScheduledPoll(
+		p: Poll,
+	): MastodonEntity.StatusParams["poll"] {
+		const now = new Date();
+		const count = p.choices.reduce((sum, choice) => sum + choice.votes, 0);
+		return {
+			expires_in: (
+				((p.expiresAt?.getTime() ?? Date.now()) - Date.now()) /
+				1000
+			).toString(),
+			multiple: p.multiple,
+			hide_totals: false,
+			options: p.choices.map((c) => c.text),
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/user.ts

@@ -0,0 +1,237 @@
+import type { ILocalUser, User } from "@/models/entities/user.js";
+import { config } from "@/config.js";
+import { DriveFiles, Followings, UserProfiles, Users } from "@/models/index.js";
+import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js";
+import { populateEmojis } from "@/misc/populate-emojis.js";
+import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
+import mfm from "mfm-js";
+import { awaitAll } from "@/prelude/await-all.js";
+import {
+	type AccountCache,
+	UserHelpers,
+} from "@/server/api/mastodon/helpers/user.js";
+import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js";
+import type { MastoContext } from "@/server/api/mastodon/index.js";
+import type { IMentionedRemoteUsers, Note } from "@/models/entities/note.js";
+import type { UserProfile } from "@/models/entities/user-profile.js";
+import { In } from "typeorm";
+import { unique } from "@/prelude/array.js";
+
+type Field = {
+	name: string;
+	value: string;
+	verified?: boolean;
+};
+
+export class UserConverter {
+	public static async encode(
+		u: User,
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Account> {
+		const localUser = ctx.user as ILocalUser | null;
+		const cache = ctx.cache as AccountCache;
+		return cache.locks.acquire(u.id, async () => {
+			const cacheHit = cache.accounts.find((p) => p.id == u.id);
+			if (cacheHit) return cacheHit;
+
+			const fqn = `${u.username}@${u.host ?? config.host}`;
+			let acct = u.username;
+			let acctUrl = `https://${u.host || config.host}/@${u.username}`;
+			if (u.host) {
+				acct = `${u.username}@${u.host}`;
+				acctUrl = `https://${u.host}/@${u.username}`;
+			}
+
+			const aggregateProfile = (
+				ctx.userProfileAggregate as Map<string, UserProfile | null>
+			)?.get(u.id);
+
+			const profile =
+				aggregateProfile !== undefined
+					? aggregateProfile
+					: UserProfiles.findOneBy({ userId: u.id });
+			const bio = Promise.resolve(profile)
+				.then(async (profile) => {
+					return MfmHelpers.toHtml(
+						mfm.parse(profile?.description ?? ""),
+						profile?.mentions,
+						u.host,
+						false,
+						null,
+						ctx,
+					)
+						.then((p) => p ?? escapeMFM(profile?.description ?? ""))
+						.then((p) => (p !== "<p></p>" ? p : null));
+				})
+				.then((p) => p ?? "<p></p>");
+
+			const avatar = u.avatarId
+				? DriveFiles.findOneBy({ id: u.avatarId })
+						.then((p) => p?.url ?? Users.getIdenticonUrl(u.id))
+						.then((p) => DriveFiles.getFinalUrl(p))
+				: Users.getIdenticonUrl(u.id);
+
+			const banner = u.bannerId
+				? DriveFiles.findOneBy({ id: u.bannerId })
+						.then(
+							(p) => p?.url ?? `${config.url}/static-assets/transparent.png`,
+						)
+						.then((p) => DriveFiles.getFinalUrl(p))
+				: `${config.url}/static-assets/transparent.png`;
+
+			const isFollowedOrSelf =
+				(ctx.followedOrSelfAggregate as Map<string, boolean>)?.get(u.id) ??
+				(!!localUser &&
+					(localUser.id === u.id ||
+						Followings.exists({
+							where: {
+								followeeId: u.id,
+								followerId: localUser.id,
+							},
+						})));
+
+			const followersCount = Promise.resolve(profile).then(async (profile) => {
+				if (profile === null) return u.followersCount;
+				switch (profile.ffVisibility) {
+					case "public":
+						return u.followersCount;
+					case "followers":
+						return Promise.resolve(isFollowedOrSelf).then((isFollowedOrSelf) =>
+							isFollowedOrSelf ? u.followersCount : 0,
+						);
+					case "private":
+						return localUser?.id === profile.userId ? u.followersCount : 0;
+				}
+			});
+
+			const followingCount = Promise.resolve(profile).then(async (profile) => {
+				if (profile === null) return u.followingCount;
+				switch (profile.ffVisibility) {
+					case "public":
+						return u.followingCount;
+					case "followers":
+						return Promise.resolve(isFollowedOrSelf).then((isFollowedOrSelf) =>
+							isFollowedOrSelf ? u.followingCount : 0,
+						);
+					case "private":
+						return localUser?.id === profile.userId ? u.followingCount : 0;
+				}
+			});
+
+			const fields = Promise.resolve(profile).then((profile) =>
+				Promise.all(
+					profile?.fields.map(async (p) =>
+						this.encodeField(p, u.host, profile?.mentions),
+					) ?? [],
+				),
+			);
+
+			return awaitAll({
+				id: u.id,
+				username: u.username,
+				acct: acct,
+				fqn: fqn,
+				display_name: u.name || u.username,
+				locked: u.isLocked,
+				created_at: u.createdAt.toISOString(),
+				followers_count: followersCount,
+				following_count: followingCount,
+				statuses_count: u.notesCount,
+				note: bio,
+				url: u.uri ?? acctUrl,
+				avatar: avatar,
+				avatar_static: avatar,
+				header: banner,
+				header_static: banner,
+				emojis: populateEmojis(u.emojis, u.host).then((emoji) =>
+					emoji.map((e) => EmojiConverter.encode(e)),
+				),
+				moved: null, //FIXME
+				fields: fields,
+				bot: u.isBot,
+				discoverable: u.isExplorable,
+			}).then((p) => {
+				UserHelpers.updateUserInBackground(u);
+				cache.accounts.push(p);
+				return p;
+			});
+		});
+	}
+
+	public static async aggregateData(
+		users: User[],
+		ctx: MastoContext,
+	): Promise<void> {
+		const user = ctx.user as ILocalUser | null;
+		const targets = unique(users.map((u) => u.id));
+
+		const followedOrSelfAggregate = new Map<User["id"], boolean>();
+		const userProfileAggregate = new Map<User["id"], UserProfile | null>();
+
+		if (user) {
+			const targetsWithoutSelf = targets.filter((u) => u !== user.id);
+
+			if (targetsWithoutSelf.length > 0) {
+				const followings = await Followings.createQueryBuilder("following")
+					.select("following.followeeId")
+					.where("following.followerId = :meId", { meId: user.id })
+					.andWhere("following.followeeId IN (:...targets)", {
+						targets: targetsWithoutSelf,
+					})
+					.getMany();
+
+				for (const userId of targetsWithoutSelf) {
+					followedOrSelfAggregate.set(
+						userId,
+						!!followings.find((f) => f.followerId === userId),
+					);
+				}
+			}
+
+			followedOrSelfAggregate.set(user.id, true);
+		}
+
+		const profiles = await UserProfiles.findBy({
+			userId: In(targets),
+		});
+
+		for (const userId of targets) {
+			userProfileAggregate.set(
+				userId,
+				profiles.find((p) => p.userId === userId) ?? null,
+			);
+		}
+
+		ctx.followedOrSelfAggregate = followedOrSelfAggregate;
+	}
+
+	public static async encodeMany(
+		users: User[],
+		ctx: MastoContext,
+	): Promise<MastodonEntity.Account[]> {
+		await this.aggregateData(users, ctx);
+		const encoded = users.map((u) => this.encode(u, ctx));
+		return Promise.all(encoded);
+	}
+
+	private static async encodeField(
+		f: Field,
+		host: string | null,
+		mentions: IMentionedRemoteUsers,
+		ctx?: MastoContext,
+	): Promise<MastodonEntity.Field> {
+		return {
+			name: f.name,
+			value:
+				(await MfmHelpers.toHtml(
+					mfm.parse(f.value),
+					mentions,
+					host,
+					true,
+					null,
+					ctx,
+				)) ?? escapeMFM(f.value),
+			verified_at: f.verified ? new Date().toISOString() : null,
+		};
+	}
+}

packages/backend/src/server/api/mastodon/converters/visibility.ts

@@ -0,0 +1,37 @@
+export type FirefishVisibility =
+	| "public"
+	| "home"
+	| "followers"
+	| "specified"
+	| "hidden";
+export type MastodonVisibility = "public" | "unlisted" | "private" | "direct";
+
+export class VisibilityConverter {
+	public static encode(v: FirefishVisibility): MastodonVisibility {
+		switch (v) {
+			case "public":
+				return v;
+			case "home":
+				return "unlisted";
+			case "followers":
+				return "private";
+			case "specified":
+				return "direct";
+			case "hidden":
+				throw new Error();
+		}
+	}
+
+	public static decode(v: MastodonVisibility): FirefishVisibility {
+		switch (v) {
+			case "public":
+				return v;
+			case "unlisted":
+				return "home";
+			case "private":
+				return "followers";
+			case "direct":
+				return "specified";
+		}
+	}
+}

packages/backend/src/server/api/mastodon/endpoints/account.ts

@@ -1,477 +1,284 @@
-import Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import { argsToBools, convertTimelinesArgsId, limitToInt } from "./timeline.js";
-import { fromMastodonId, toMastodonId } from "backend-rs";
-import {
-	convertAccount,
-	convertFeaturedTag,
-	convertList,
-	convertRelationship,
-	convertStatus,
-} from "../converters.js";
-import { apiLogger } from "../../logger.js";
-import { inspect } from "node:util";
+import type Router from "@koa/router";
+import { argsToBools, limitToInt, normalizeUrlQuery } from "./timeline.js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
+import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
+import { ListHelpers } from "@/server/api/mastodon/helpers/list.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { SearchHelpers } from "@/server/api/mastodon/helpers/search.js";
+import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
 
-const relationshipModel = {
-	id: "",
-	following: false,
-	followed_by: false,
-	delivery_following: false,
-	blocking: false,
-	blocked_by: false,
-	muting: false,
-	muting_notifications: false,
-	requested: false,
-	domain_blocking: false,
-	showing_reblogs: false,
-	endorsed: false,
-	notifying: false,
-	note: "",
-};
-
-export function apiAccountMastodon(router: Router): void {
-	router.get("/v1/accounts/verify_credentials", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.verifyAccountCredentials();
-			let acct = data.data;
-			acct.id = toMastodonId(acct.id);
-			acct.display_name = acct.display_name || acct.username;
-			acct.url = `${BASE_URL}/@${acct.url}`;
-			acct.note = acct.note || "";
-			acct.avatar_static = acct.avatar;
-			acct.header = acct.header || "/static-assets/transparent.png";
-			acct.header_static = acct.header || "/static-assets/transparent.png";
-			acct.source = {
-				note: acct.note,
-				fields: acct.fields,
-				privacy: await client.getDefaultPostPrivacy(),
-				sensitive: false,
-				language: "",
-			};
-			console.log(acct);
-			ctx.body = acct;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.patch("/v1/accounts/update_credentials", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.updateCredentials(
-				(ctx.request as any).body as any,
-			);
-			ctx.body = convertAccount(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+export function setupEndpointsAccount(router: Router): void {
+	router.get(
+		"/v1/accounts/verify_credentials",
+		auth(true, ["read:accounts"]),
+		async (ctx) => {
+			ctx.body = await UserHelpers.verifyCredentials(ctx);
+		},
+	);
+	router.patch(
+		"/v1/accounts/update_credentials",
+		auth(true, ["write:accounts"]),
+		async (ctx) => {
+			ctx.body = await UserHelpers.updateCredentials(ctx);
+		},
+	);
 	router.get("/v1/accounts/lookup", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.search(
-				(ctx.request.query as any).acct,
+		const args = normalizeUrlQuery(ctx.query);
+		const user = await UserHelpers.getUserFromAcct(args.acct);
+		ctx.body = await UserConverter.encode(user, ctx);
+	});
+	router.get(
+		"/v1/accounts/relationships",
+		auth(true, ["read:follows"]),
+		async (ctx) => {
+			const ids =
+				normalizeUrlQuery(ctx.query, ["id[]"])["id[]"] ??
+				normalizeUrlQuery(ctx.query, ["id"])["id"] ??
+				[];
+			ctx.body = await UserHelpers.getUserRelationhipToMany(ids, ctx.user.id);
+		},
+	);
+	// This must come before /accounts/:id, otherwise that will take precedence
+	router.get(
+		"/v1/accounts/search",
+		auth(true, ["read:accounts"]),
+		async (ctx) => {
+			const args = normalizeUrlQuery(
+				argsToBools(limitToInt(ctx.query), ["resolve", "following"]),
+			);
+			ctx.body = await SearchHelpers.search(
+				args.q,
 				"accounts",
+				args.resolve,
+				args.following,
+				undefined,
+				false,
+				undefined,
+				undefined,
+				args.limit,
+				args.offset,
+				ctx,
+			).then((p) => p.accounts);
+		},
+	);
+	router.get<{ Params: { id: string } }>(
+		"/v1/accounts/:id",
+		auth(false),
+		async (ctx) => {
+			ctx.body = await UserConverter.encode(
+				await UserHelpers.getUserOr404(ctx.params.id),
+				ctx,
 			);
-			ctx.body = convertAccount(data.data.accounts[0]);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/accounts/relationships", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		let users;
-		try {
-			// TODO: this should be body
-			let ids = ctx.request.query ? ctx.request.query["id[]"] : null;
-			if (typeof ids === "string") {
-				ids = [ids];
-			}
-			users = ids;
-			relationshipModel.id = ids?.toString() || "1";
-			if (!ids) {
-				ctx.body = [relationshipModel];
-				return;
-			}
-
-			let reqIds = [];
-			for (let i = 0; i < ids.length; i++) {
-				reqIds.push(fromMastodonId(ids[i]));
-			}
-
-			const data = await client.getRelationships(reqIds);
-			ctx.body = data.data.map((relationship) =>
-				convertRelationship(relationship),
-			);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			let data = e.response.data;
-			data.users = users;
-			ctx.status = 401;
-			ctx.body = data;
-		}
-	});
-	router.get<{ Params: { id: string } }>("/v1/accounts/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const calcId = fromMastodonId(ctx.params.id);
-			const data = await client.getAccount(calcId);
-			ctx.body = convertAccount(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+		},
+	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/accounts/:id/statuses",
+		auth(false, ["read:statuses"]),
+		filterContext("account"),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountStatuses(
-					fromMastodonId(ctx.params.id),
-					convertTimelinesArgsId(argsToBools(limitToInt(ctx.query as any))),
-				);
-				ctx.body = data.data.map((status) => convertStatus(status));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const query = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)));
+			const res = await UserHelpers.getUserStatuses(
+				query,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				args.only_media,
+				args.exclude_replies,
+				args.exclude_reblogs,
+				args.pinned,
+				args.tagged,
+				ctx,
+			);
+			ctx.body = await NoteConverter.encodeMany(res, ctx);
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/accounts/:id/featured_tags",
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountFeaturedTags(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data.map((tag) => convertFeaturedTag(tag));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			ctx.body = [];
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/accounts/:id/followers",
+		auth(false),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountFollowers(
-					fromMastodonId(ctx.params.id),
-					convertTimelinesArgsId(limitToInt(ctx.query as any)),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const query = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+			const res = await UserHelpers.getUserFollowers(
+				query,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await UserConverter.encodeMany(res, ctx);
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/accounts/:id/following",
+		auth(false),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountFollowing(
-					fromMastodonId(ctx.params.id),
-					convertTimelinesArgsId(limitToInt(ctx.query as any)),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const query = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+			const res = await UserHelpers.getUserFollowing(
+				query,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await UserConverter.encodeMany(res, ctx);
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/accounts/:id/lists",
+		auth(true, ["read:lists"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountLists(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data.map((list) => convertList(list));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const member = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await ListHelpers.getListsByMember(member, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/follow",
+		auth(true, ["write:follows"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.followAccount(fromMastodonId(ctx.params.id));
-				let acct = convertRelationship(data.data);
-				acct.following = true;
-				ctx.body = acct;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			// FIXME: Parse form data
+			ctx.body = await UserHelpers.followUser(target, true, false, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/unfollow",
+		auth(true, ["write:follows"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unfollowAccount(
-					fromMastodonId(ctx.params.id),
-				);
-				let acct = convertRelationship(data.data);
-				acct.following = false;
-				ctx.body = acct;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.unfollowUser(target, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/block",
+		auth(true, ["write:blocks"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.blockAccount(fromMastodonId(ctx.params.id));
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.blockUser(target, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/unblock",
+		auth(true, ["write:blocks"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unblockAccount(toMastodonId(ctx.params.id));
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.unblockUser(target, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/mute",
+		auth(true, ["write:mutes"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.muteAccount(
-					fromMastodonId(ctx.params.id),
-					(ctx.request as any).body as any,
-				);
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			// FIXME: parse form data
+			const args = normalizeUrlQuery(
+				argsToBools(limitToInt(ctx.query, ["duration"]), ["notifications"]),
+			);
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.muteUser(
+				target,
+				args.notifications,
+				args.duration,
+				ctx,
+			);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/accounts/:id/unmute",
+		auth(true, ["write:mutes"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unmuteAccount(fromMastodonId(ctx.params.id));
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.unmuteUser(target, ctx);
 		},
 	);
 	router.get("/v1/featured_tags", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getFeaturedTags();
-			ctx.body = data.data.map((tag) => convertFeaturedTag(tag));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+		ctx.body = [];
 	});
 	router.get("/v1/followed_tags", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getFollowedTags();
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+		ctx.body = [];
 	});
-	router.get("/v1/bookmarks", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getBookmarks(
-				convertTimelinesArgsId(limitToInt(ctx.query as any)),
+	router.get("/v1/bookmarks", auth(true, ["read:bookmarks"]), async (ctx) => {
+		const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+		const res = await UserHelpers.getUserBookmarks(
+			args.max_id,
+			args.since_id,
+			args.min_id,
+			args.limit,
+			ctx,
+		);
+		ctx.body = await NoteConverter.encodeMany(res, ctx);
+	});
+	router.get("/v1/favourites", auth(true, ["read:favourites"]), async (ctx) => {
+		const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+		const res = await UserHelpers.getUserFavorites(
+			args.max_id,
+			args.since_id,
+			args.min_id,
+			args.limit,
+			ctx,
+		);
+		ctx.body = await NoteConverter.encodeMany(res, ctx);
+	});
+	router.get("/v1/mutes", auth(true, ["read:mutes"]), async (ctx) => {
+		const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+		ctx.body = await UserHelpers.getUserMutes(
+			args.max_id,
+			args.since_id,
+			args.min_id,
+			args.limit,
+			ctx,
+		);
+	});
+	router.get("/v1/blocks", auth(true, ["read:blocks"]), async (ctx) => {
+		const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+		const res = await UserHelpers.getUserBlocks(
+			args.max_id,
+			args.since_id,
+			args.min_id,
+			args.limit,
+			ctx,
+		);
+		ctx.body = await UserConverter.encodeMany(res, ctx);
+	});
+	router.get(
+		"/v1/follow_requests",
+		auth(true, ["read:follows"]),
+		async (ctx) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+			const res = await UserHelpers.getUserFollowRequests(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
 			);
-			ctx.body = data.data.map((status) => convertStatus(status));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/favourites", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getFavourites(
-				convertTimelinesArgsId(limitToInt(ctx.query as any)),
-			);
-			ctx.body = data.data.map((status) => convertStatus(status));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/mutes", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getMutes(
-				convertTimelinesArgsId(limitToInt(ctx.query as any)),
-			);
-			ctx.body = data.data.map((account) => convertAccount(account));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/blocks", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getBlocks(
-				convertTimelinesArgsId(limitToInt(ctx.query as any)),
-			);
-			ctx.body = data.data.map((account) => convertAccount(account));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/follow_requests", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getFollowRequests(
-				((ctx.query as any) || { limit: 20 }).limit,
-			);
-			ctx.body = data.data.map((account) => convertAccount(account));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+			ctx.body = await UserConverter.encodeMany(res, ctx);
+		},
+	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/follow_requests/:id/authorize",
+		auth(true, ["write:follows"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.acceptFollowRequest(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.acceptFollowRequest(target, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/follow_requests/:id/reject",
+		auth(true, ["write:follows"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.rejectFollowRequest(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = convertRelationship(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const target = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
+			ctx.body = await UserHelpers.rejectFollowRequest(target, ctx);
 		},
 	);
 }

packages/backend/src/server/api/mastodon/endpoints/auth.ts

@@ -1,80 +1,31 @@
-import Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+import type Router from "@koa/router";
+import { AuthHelpers } from "@/server/api/mastodon/helpers/auth.js";
+import { MiAuth } from "@/server/api/mastodon/middleware/auth.js";
 
-const readScope = [
-	"read:account",
-	"read:drive",
-	"read:blocks",
-	"read:favorites",
-	"read:following",
-	"read:messaging",
-	"read:mutes",
-	"read:notifications",
-	"read:reactions",
-	"read:pages",
-	"read:page-likes",
-	"read:user-groups",
-	"read:channels",
-	"read:gallery",
-	"read:gallery-likes",
-];
-const writeScope = [
-	"write:account",
-	"write:drive",
-	"write:blocks",
-	"write:favorites",
-	"write:following",
-	"write:messaging",
-	"write:mutes",
-	"write:notes",
-	"write:notifications",
-	"write:reactions",
-	"write:votes",
-	"write:pages",
-	"write:page-likes",
-	"write:user-groups",
-	"write:channels",
-	"write:gallery",
-	"write:gallery-likes",
-];
-
-export function apiAuthMastodon(router: Router): void {
+export function setupEndpointsAuth(router: Router): void {
 	router.post("/v1/apps", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const client = getClient(BASE_URL, "");
-		const body: any = ctx.request.body || ctx.request.query;
-		try {
-			let scope = body.scopes;
-			if (typeof scope === "string") scope = scope.split(" ");
-			const pushScope = new Set<string>();
-			for (const s of scope) {
-				if (s.match(/^read/)) for (const r of readScope) pushScope.add(r);
-				if (s.match(/^write/)) for (const r of writeScope) pushScope.add(r);
-			}
-			const scopeArr = Array.from(pushScope);
+		ctx.body = await AuthHelpers.registerApp(ctx);
+	});
 
-			const red = body.redirect_uris;
-			const appData = await client.registerApp(body.client_name, {
-				scopes: scopeArr,
-				redirect_uris: red,
-				website: body.website,
-			});
-			const returns = {
-				id: Math.floor(Math.random() * 100).toString(),
-				name: appData.name,
-				website: body.website,
-				redirect_uri: red,
-				client_id: Buffer.from(appData.url || "").toString("base64"),
-				client_secret: appData.clientSecret,
-			};
-			console.log(returns);
-			ctx.body = returns;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+	router.get("/v1/apps/verify_credentials", async (ctx) => {
+		ctx.body = await AuthHelpers.verifyAppCredentials(ctx);
+	});
+
+	router.post("/v1/firefish/apps/info", MiAuth(true), async (ctx) => {
+		ctx.body = await AuthHelpers.getAppInfo(ctx);
+	});
+
+	router.post("/v1/firefish/auth/code", MiAuth(true), async (ctx) => {
+		ctx.body = await AuthHelpers.getAuthCode(ctx);
+	});
+}
+
+export function setupEndpointsAuthRoot(router: Router): void {
+	router.post("/oauth/token", async (ctx) => {
+		ctx.body = await AuthHelpers.getAuthToken(ctx);
+	});
+
+	router.post("/oauth/revoke", async (ctx) => {
+		ctx.body = await AuthHelpers.revokeAuthToken(ctx);
 	});
 }

packages/backend/src/server/api/mastodon/endpoints/filter.ts

@@ -1,87 +1,23 @@
-import Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import { IdType, convertId } from "@/server/api/index.js";
-import { convertFilter } from "../converters.js";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+import type Router from "@koa/router";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
 
-export function apiFilterMastodon(router: Router): void {
-	router.get("/v1/filters", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.getFilters();
-			ctx.body = data.data.map((filter) => convertFilter(filter));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.get("/v1/filters/:id", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.getFilter(fromMastodonId(ctx.params.id));
-			ctx.body = convertFilter(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.post("/v1/filters", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.createFilter(body.phrase, body.context, body);
-			ctx.body = convertFilter(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.post("/v1/filters/:id", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.updateFilter(
-				fromMastodonId(ctx.params.id),
-				body.phrase,
-				body.context,
+export function setupEndpointsFilter(router: Router): void {
+	router.get(
+		["/v1/filters", "/v2/filters"],
+		auth(true, ["read:filters"]),
+		async (ctx) => {
+			ctx.body = [];
+		},
+	);
+	router.post(
+		["/v1/filters", "/v2/filters"],
+		auth(true, ["write:filters"]),
+		async (ctx) => {
+			throw new MastoApiError(
+				400,
+				"Please change word mute settings in the web frontend settings.",
 			);
-			ctx.body = convertFilter(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-
-	router.delete("/v1/filters/:id", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.deleteFilter(fromMastodonId(ctx.params.id));
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+		},
+	);
 }

packages/backend/src/server/api/mastodon/endpoints/list.ts

@@ -0,0 +1,116 @@
+import type Router from "@koa/router";
+import {
+	limitToInt,
+	normalizeUrlQuery,
+} from "@/server/api/mastodon/endpoints/timeline.js";
+import { ListHelpers } from "@/server/api/mastodon/helpers/list.js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { UserLists } from "@/models/index.js";
+import { getUser } from "@/server/api/common/getters.js";
+import { toArray } from "@/prelude/array.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
+
+export function setupEndpointsList(router: Router): void {
+	router.get("/v1/lists", auth(true, ["read:lists"]), async (ctx, reply) => {
+		ctx.body = await ListHelpers.getLists(ctx);
+	});
+	router.get<{ Params: { id: string } }>(
+		"/v1/lists/:id",
+		auth(true, ["read:lists"]),
+		async (ctx, reply) => {
+			ctx.body = await ListHelpers.getListOr404(ctx.params.id, ctx);
+		},
+	);
+	router.post("/v1/lists", auth(true, ["write:lists"]), async (ctx, reply) => {
+		const body = ctx.request.body as any;
+		const title = (body.title ?? "").trim();
+		ctx.body = await ListHelpers.createList(title, ctx);
+	});
+	router.put<{ Params: { id: string } }>(
+		"/v1/lists/:id",
+		auth(true, ["write:lists"]),
+		async (ctx, reply) => {
+			const list = await UserLists.findOneBy({
+				userId: ctx.user.id,
+				id: ctx.params.id,
+			});
+			if (!list) throw new MastoApiError(404);
+
+			const body = ctx.request.body as any;
+			const title = (body.title ?? "").trim();
+			const exclusive = body.exclusive ?? (undefined as boolean | undefined);
+			ctx.body = await ListHelpers.updateList(list, title, exclusive, ctx);
+		},
+	);
+	router.delete<{ Params: { id: string } }>(
+		"/v1/lists/:id",
+		auth(true, ["write:lists"]),
+		async (ctx, reply) => {
+			const list = await UserLists.findOneBy({
+				userId: ctx.user.id,
+				id: ctx.params.id,
+			});
+			if (!list) throw new MastoApiError(404);
+
+			await ListHelpers.deleteList(list, ctx);
+			ctx.body = {};
+		},
+	);
+	router.get<{ Params: { id: string } }>(
+		"/v1/lists/:id/accounts",
+		auth(true, ["read:lists"]),
+		async (ctx, reply) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query));
+			const res = await ListHelpers.getListUsers(
+				ctx.params.id,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await UserConverter.encodeMany(res, ctx);
+		},
+	);
+	router.post<{ Params: { id: string } }>(
+		"/v1/lists/:id/accounts",
+		auth(true, ["write:lists"]),
+		async (ctx, reply) => {
+			const list = await UserLists.findOneBy({
+				userId: ctx.user.id,
+				id: ctx.params.id,
+			});
+			if (!list) throw new MastoApiError(404);
+
+			const body = ctx.request.body as any;
+			if (!body.account_ids)
+				throw new MastoApiError(400, "Missing account_ids[] field");
+
+			const ids = toArray(body.account_ids);
+			const targets = await Promise.all(ids.map((p) => getUser(p)));
+			await ListHelpers.addToList(list, targets, ctx);
+			ctx.body = {};
+		},
+	);
+	router.delete<{ Params: { id: string } }>(
+		"/v1/lists/:id/accounts",
+		auth(true, ["write:lists"]),
+		async (ctx, reply) => {
+			const list = await UserLists.findOneBy({
+				userId: ctx.user.id,
+				id: ctx.params.id,
+			});
+			if (!list) throw new MastoApiError(404);
+
+			const body = ctx.request.body as any;
+			if (!body.account_ids)
+				throw new MastoApiError(400, "Missing account_ids[] field");
+
+			const ids = toArray(body.account_ids);
+			const targets = await Promise.all(ids.map((p) => getUser(p)));
+			await ListHelpers.removeFromList(list, targets, ctx);
+			ctx.body = {};
+		},
+	);
+}

packages/backend/src/server/api/mastodon/endpoints/media.ts

@@ -0,0 +1,34 @@
+import type Router from "@koa/router";
+import { MediaHelpers } from "@/server/api/mastodon/helpers/media.js";
+import { FileConverter } from "@/server/api/mastodon/converters/file.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+
+export function setupEndpointsMedia(router: Router): void {
+	router.get<{ Params: { id: string } }>(
+		"/v1/media/:id",
+		auth(true, ["write:media"]),
+		async (ctx) => {
+			const file = await MediaHelpers.getMediaPackedOr404(ctx.params.id, ctx);
+			ctx.body = FileConverter.encode(file);
+		},
+	);
+	router.put<{ Params: { id: string } }>(
+		"/v1/media/:id",
+		auth(true, ["write:media"]),
+		async (ctx) => {
+			const file = await MediaHelpers.getMediaOr404(ctx.params.id, ctx);
+			ctx.body = await MediaHelpers.updateMedia(file, ctx).then((p) =>
+				FileConverter.encode(p),
+			);
+		},
+	);
+	router.post(
+		["/v2/media", "/v1/media"],
+		auth(true, ["write:media"]),
+		async (ctx) => {
+			ctx.body = await MediaHelpers.uploadMedia(ctx).then((p) =>
+				FileConverter.encode(p),
+			);
+		},
+	);
+}

packages/backend/src/server/api/mastodon/endpoints/meta.ts

@@ -1,68 +0,0 @@
-import type { Entity } from "megalodon";
-import { config } from "@/config.js";
-import { FILE_TYPE_BROWSERSAFE, fetchMeta } from "backend-rs";
-import { Users, Notes } from "@/models/index.js";
-import { IsNull } from "typeorm";
-
-export async function getInstance(
-	response: Entity.Instance,
-	contact: Entity.Account,
-) {
-	const [meta, totalUsers, totalStatuses] = await Promise.all([
-		fetchMeta(true),
-		Users.count({ where: { host: IsNull() } }),
-		Notes.count({ where: { userHost: IsNull() } }),
-	]);
-
-	return {
-		uri: response.uri,
-		title: response.title || "Firefish",
-		short_description:
-			response.description?.substring(0, 50) || "See real server website",
-		description:
-			response.description ||
-			"This is a vanilla Firefish Instance. It doesn't seem to have a description.",
-		email: response.email || "",
-		version: `3.0.0 (compatible; Firefish ${config.version})`,
-		urls: response.urls,
-		stats: {
-			user_count: totalUsers,
-			status_count: totalStatuses,
-			domain_count: response.stats.domain_count,
-		},
-		thumbnail: response.thumbnail || "/static-assets/transparent.png",
-		languages: meta.langs,
-		registrations: !meta.disableRegistration || response.registrations,
-		approval_required: !response.registrations,
-		invites_enabled: response.registrations,
-		configuration: {
-			accounts: {
-				max_featured_tags: 20,
-			},
-			statuses: {
-				max_characters: config.maxNoteLength,
-				max_media_attachments: 16,
-				characters_reserved_per_url: response.uri.length,
-			},
-			media_attachments: {
-				supported_mime_types: FILE_TYPE_BROWSERSAFE,
-				image_size_limit: 10485760,
-				image_matrix_limit: 16777216,
-				video_size_limit: 41943040,
-				video_frame_rate_limit: 60,
-				video_matrix_limit: 2304000,
-			},
-			polls: {
-				max_options: 10,
-				max_characters_per_option: 50,
-				min_expiration: 50,
-				max_expiration: 2629746,
-			},
-			reactions: {
-				max_reactions: 1,
-			},
-		},
-		contact_account: contact,
-		rules: [],
-	};
-}

packages/backend/src/server/api/mastodon/endpoints/misc.ts

@@ -0,0 +1,70 @@
+import type Router from "@koa/router";
+import { MiscHelpers } from "@/server/api/mastodon/helpers/misc.js";
+import {
+	argsToBools,
+	limitToInt,
+} from "@/server/api/mastodon/endpoints/timeline.js";
+import { Announcements } from "@/models/index.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
+import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
+
+export function setupEndpointsMisc(router: Router): void {
+	router.get("/v1/custom_emojis", async (ctx) => {
+		ctx.body = await MiscHelpers.getCustomEmoji();
+	});
+
+	router.get("/v1/instance", async (ctx) => {
+		ctx.body = await MiscHelpers.getInstance(ctx);
+	});
+
+	router.get("/v2/instance", async (ctx) => {
+		ctx.body = await MiscHelpers.getInstanceV2(ctx);
+	});
+
+	router.get("/v1/announcements", auth(true), async (ctx) => {
+		const args = argsToBools(ctx.query, ["with_dismissed"]);
+		ctx.body = await MiscHelpers.getAnnouncements(args.with_dismissed, ctx);
+	});
+
+	router.post<{ Params: { id: string } }>(
+		"/v1/announcements/:id/dismiss",
+		auth(true, ["write:accounts"]),
+		async (ctx) => {
+			const announcement = await Announcements.findOneBy({ id: ctx.params.id });
+			if (!announcement) throw new MastoApiError(404);
+
+			await MiscHelpers.dismissAnnouncement(announcement, ctx);
+			ctx.body = {};
+		},
+	);
+
+	// FIXME: add link pagination to trends (ref: https://mastodon.social/api/v1/trends/tags?offset=10&limit=1)
+	router.get(["/v1/trends/tags", "/v1/trends"], async (ctx) => {
+		const args = limitToInt(ctx.query);
+		ctx.body = await MiscHelpers.getTrendingHashtags(args.limit, args.offset);
+		// FIXME: convert ids
+	});
+
+	router.get("/v1/trends/statuses", filterContext("public"), async (ctx) => {
+		const args = limitToInt(ctx.query);
+		ctx.body = await MiscHelpers.getTrendingStatuses(
+			args.limit,
+			args.offset,
+			ctx,
+		);
+	});
+
+	router.get("/v1/trends/links", async (ctx) => {
+		ctx.body = [];
+	});
+
+	router.get("/v1/preferences", auth(true, ["read:accounts"]), async (ctx) => {
+		ctx.body = await MiscHelpers.getPreferences(ctx);
+	});
+
+	router.get("/v2/suggestions", auth(true, ["read:accounts"]), async (ctx) => {
+		const args = limitToInt(ctx.query);
+		ctx.body = await MiscHelpers.getFollowSuggestions(args.limit, ctx);
+	});
+}

packages/backend/src/server/api/mastodon/endpoints/notifications.ts

@@ -1,98 +1,107 @@
-import Router from "@koa/router";
-import { fromMastodonId } from "backend-rs";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import { convertTimelinesArgsId } from "./timeline.js";
-import { convertNotification } from "../converters.js";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+import type Router from "@koa/router";
+import { limitToInt, normalizeUrlQuery } from "./timeline.js";
+import { NotificationHelpers } from "@/server/api/mastodon/helpers/notification.js";
+import { NotificationConverter } from "@/server/api/mastodon/converters/notification.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
 
-function toLimitToInt(q: any) {
-	if (q.limit) if (typeof q.limit === "string") q.limit = parseInt(q.limit, 10);
-	return q;
-}
-
-export function apiNotificationsMastodon(router: Router): void {
-	router.get("/v1/notifications", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.getNotifications(
-				convertTimelinesArgsId(toLimitToInt(ctx.query)),
+export function setupEndpointsNotifications(router: Router): void {
+	router.get(
+		"/v1/notifications",
+		auth(true, ["read:notifications"]),
+		filterContext("notifications"),
+		async (ctx) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query), [
+				"types[]",
+				"exclude_types[]",
+			]);
+			const res = await NotificationHelpers.getNotifications(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				args["types[]"],
+				args["exclude_types[]"],
+				args.account_id,
+				ctx,
 			);
-			const notfs = data.data;
-			const ret = notfs.map((n) => {
-				n = convertNotification(n);
-				if (n.type !== "follow" && n.type !== "follow_request") {
-					if (n.type === "reaction") n.type = "favourite";
-					return n;
-				} else {
-					return n;
-				}
-			});
-			ctx.body = ret;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+			ctx.body = await NotificationConverter.encodeMany(res, ctx);
+		},
+	);
+
+	router.get(
+		"/v1/notifications/:id",
+		auth(true, ["read:notifications"]),
+		filterContext("notifications"),
+		async (ctx) => {
+			const notification = await NotificationHelpers.getNotificationOr404(
+				ctx.params.id,
+				ctx,
+			);
+			ctx.body = await NotificationConverter.encode(notification, ctx);
+		},
+	);
+
+	router.post(
+		"/v1/notifications/clear",
+		auth(true, ["write:notifications"]),
+		async (ctx) => {
+			await NotificationHelpers.clearAllNotifications(ctx);
+			ctx.body = {};
+		},
+	);
+
+	router.post(
+		"/v1/notifications/:id/dismiss",
+		auth(true, ["write:notifications"]),
+		async (ctx) => {
+			const notification = await NotificationHelpers.getNotificationOr404(
+				ctx.params.id,
+				ctx,
+			);
+			await NotificationHelpers.dismissNotification(notification.id, ctx);
+			ctx.body = {};
+		},
+	);
+
+	router.post(
+		"/v1/conversations/:id/read",
+		auth(true, ["write:conversations"]),
+		async (ctx, reply) => {
+			await NotificationHelpers.markConversationAsRead(ctx.params.id, ctx);
+			ctx.body = {};
+		},
+	);
+
+	router.get("/v1/push/subscription", auth(true, ["push"]), async (ctx) => {
+		const subscription =
+			await NotificationHelpers.getPushSubscriptionOr404(ctx);
+		ctx.body = await NotificationConverter.encodeSubscription(
+			subscription,
+			ctx,
+		);
 	});
 
-	router.get("/v1/notification/:id", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const dataRaw = await client.getNotification(
-				fromMastodonId(ctx.params.id),
-			);
-			const data = convertNotification(dataRaw.data);
-			ctx.body = data;
-			if (
-				data.type !== "follow" &&
-				data.type !== "follow_request" &&
-				data.type === "reaction"
-			) {
-				data.type = "favourite";
-			}
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+	router.post("/v1/push/subscription", auth(true, ["push"]), async (ctx) => {
+		const subscription = await NotificationHelpers.setPushSubscription(ctx);
+		ctx.body = await NotificationConverter.encodeSubscription(
+			subscription,
+			ctx,
+		);
 	});
 
-	router.post("/v1/notifications/clear", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.dismissNotifications();
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+	router.delete("/v1/push/subscription", auth(true, ["push"]), async (ctx) => {
+		await NotificationHelpers.deletePushSubscription(ctx);
+		ctx.body = {};
 	});
 
-	router.post("/v1/notification/:id/dismiss", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const data = await client.dismissNotification(
-				fromMastodonId(ctx.params.id),
-			);
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
+	router.put("/v1/push/subscription", auth(true, ["push"]), async (ctx) => {
+		const subscription =
+			await NotificationHelpers.getPushSubscriptionOr404(ctx);
+		await NotificationHelpers.putPushSubscription(subscription, ctx);
+		ctx.body = await NotificationConverter.encodeSubscription(
+			subscription,
+			ctx,
+		);
 	});
 }

packages/backend/src/server/api/mastodon/endpoints/search.ts

@@ -1,146 +1,43 @@
-import Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import axios from "axios";
-import { Converter } from "megalodon";
-import { convertTimelinesArgsId, limitToInt } from "./timeline.js";
-import { convertAccount, convertStatus } from "../converters.js";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+import type Router from "@koa/router";
+import { argsToBools, limitToInt, normalizeUrlQuery } from "./timeline.js";
+import { SearchHelpers } from "@/server/api/mastodon/helpers/search.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
 
-export function apiSearchMastodon(router: Router): void {
-	router.get("/v1/search", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.request.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		const body: any = ctx.request.body;
-		try {
-			const query: any = convertTimelinesArgsId(limitToInt(ctx.query));
-			const type = query.type || "";
-			const data = await client.search(query.q, type, query);
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v2/search", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const query: any = convertTimelinesArgsId(limitToInt(ctx.query));
-			const type = query.type;
-			const acct =
-				!type || type === "accounts"
-					? await client.search(query.q, "accounts", query)
-					: null;
-			const stat =
-				!type || type === "statuses"
-					? await client.search(query.q, "statuses", query)
-					: null;
-			const tags =
-				!type || type === "hashtags"
-					? await client.search(query.q, "hashtags", query)
-					: null;
+export function setupEndpointsSearch(router: Router): void {
+	router.get(
+		["/v1/search", "/v2/search"],
+		auth(true, ["read:search"]),
+		async (ctx) => {
+			const args = normalizeUrlQuery(
+				argsToBools(limitToInt(ctx.query), [
+					"resolve",
+					"following",
+					"exclude_unreviewed",
+				]),
+			);
+			const body = await SearchHelpers.search(
+				args.q,
+				args.type,
+				args.resolve,
+				args.following,
+				args.account_id,
+				args.exclude_unreviewed,
+				args.max_id,
+				args.min_id,
+				args.limit,
+				args.offset,
+				ctx,
+			);
 
-			ctx.body = {
-				accounts:
-					acct?.data?.accounts.map((account) => convertAccount(account)) ?? [],
-				statuses:
-					stat?.data?.statuses.map((status) => convertStatus(status)) ?? [],
-				hashtags: tags?.data?.hashtags ?? [],
-			};
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/trends/statuses", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		try {
-			const data = await getHighlight(
-				BASE_URL,
-				ctx.request.hostname,
-				accessTokens,
-			);
-			ctx.body = data.map((status) => convertStatus(status));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v2/suggestions", async (ctx) => {
-		const BASE_URL = `${ctx.request.protocol}://${ctx.request.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		try {
-			const query: any = ctx.query;
-			let data = await getFeaturedUser(
-				BASE_URL,
-				ctx.request.hostname,
-				accessTokens,
-				query.limit || 20,
-			);
-			data = data.map((suggestion) => {
-				suggestion.account = convertAccount(suggestion.account);
-				return suggestion;
-			});
-			console.log(data);
-			ctx.body = data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-}
-async function getHighlight(
-	BASE_URL: string,
-	domain: string,
-	accessTokens: string | undefined,
-) {
-	const accessTokenArr = accessTokens?.split(" ") ?? [null];
-	const accessToken = accessTokenArr[accessTokenArr.length - 1];
-	try {
-		const api = await axios.post(`${BASE_URL}/api/notes/featured`, {
-			i: accessToken,
-		});
-		const data: MisskeyEntity.Note[] = api.data;
-		return data.map((note) => new Converter(BASE_URL).note(note, domain));
-	} catch (e: any) {
-		apiLogger.info(inspect(e));
-		return [];
-	}
-}
-async function getFeaturedUser(
-	BASE_URL: string,
-	host: string,
-	accessTokens: string | undefined,
-	limit: number,
-) {
-	const accessTokenArr = accessTokens?.split(" ") ?? [null];
-	const accessToken = accessTokenArr[accessTokenArr.length - 1];
-	try {
-		const api = await axios.post(`${BASE_URL}/api/users`, {
-			i: accessToken,
-			limit,
-			origin: "local",
-			sort: "+follower",
-			state: "alive",
-		});
-		const data: MisskeyEntity.UserDetail[] = api.data;
-		apiLogger.info(inspect(data));
-		return data.map((u) => {
-			return {
-				source: "past_interactions",
-				account: new Converter(BASE_URL).userDetail(u, host),
-			};
-		});
-	} catch (e: any) {
-		apiLogger.info(inspect(e));
-		return [];
-	}
+			if (ctx.path === "/v1/search") {
+				const v1Body = {
+					...body,
+					hashtags: body.hashtags.map((p: MastodonEntity.Tag) => p.name),
+				};
+				ctx.body = v1Body;
+			} else {
+				ctx.body = body;
+			}
+		},
+	);
 }

packages/backend/src/server/api/mastodon/endpoints/status.ts

@@ -1,460 +1,378 @@
-import Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
-import querystring from "node:querystring";
-import qs from "qs";
-import { convertTimelinesArgsId, limitToInt } from "./timeline.js";
-import { fetchMeta, fromMastodonId, isUnicodeEmoji } from "backend-rs";
+import type Router from "@koa/router";
+import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
+import { NoteHelpers } from "@/server/api/mastodon/helpers/note.js";
 import {
-	convertAccount,
-	convertAttachment,
-	convertPoll,
-	convertStatus,
-} from "../converters.js";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+	limitToInt,
+	normalizeUrlQuery,
+} from "@/server/api/mastodon/endpoints/timeline.js";
+import { UserConverter } from "@/server/api/mastodon/converters/user.js";
+import { PollHelpers } from "@/server/api/mastodon/helpers/poll.js";
+import { toArray } from "@/prelude/array.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
+import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
 
-function normalizeQuery(data: any) {
-	const str = querystring.stringify(data);
-	return qs.parse(str);
-}
+export function setupEndpointsStatus(router: Router): void {
+	router.post("/v1/statuses", auth(true, ["write:statuses"]), async (ctx) => {
+		const key = NoteHelpers.getIdempotencyKey(ctx);
+		if (key !== null) {
+			const result = await NoteHelpers.getFromIdempotencyCache(key);
 
-export function apiStatusMastodon(router: Router): void {
-	router.post("/v1/statuses", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			let body: any = ctx.request.body;
-			if (body.in_reply_to_id)
-				body.in_reply_to_id = fromMastodonId(body.in_reply_to_id);
-			if (body.quote_id) body.quote_id = fromMastodonId(body.quote_id);
-			if (
-				(!body.poll && body["poll[options][]"]) ||
-				(!body.media_ids && body["media_ids[]"])
-			) {
-				body = normalizeQuery(body);
+			if (result) {
+				ctx.body = result;
+				return;
 			}
-			const text = body.status;
-			const removed = text.replace(/@\S+/g, "").replace(/\s|​/g, "");
-			const isDefaultEmoji = isUnicodeEmoji(removed);
-			const isCustomEmoji = /^:[a-zA-Z0-9@_]+:$/.test(removed);
-			if ((body.in_reply_to_id && isDefaultEmoji) || isCustomEmoji) {
-				const a = await client.createEmojiReaction(
-					body.in_reply_to_id,
-					removed,
-				);
-				ctx.body = a.data;
-			}
-			if (body.in_reply_to_id && removed === "/unreact") {
-				try {
-					const id = body.in_reply_to_id;
-					const post = await client.getStatus(id);
-					const react = post.data.reactions.filter((e) => e.me)[0].name;
-					const data = await client.deleteEmojiReaction(id, react);
-					ctx.body = data.data;
-				} catch (e: any) {
-					apiLogger.error(inspect(e));
-					ctx.status = 401;
-					ctx.body = e.response.data;
-				}
-			}
-			if (!body.media_ids) body.media_ids = undefined;
-			if (body.media_ids && !body.media_ids.length) body.media_ids = undefined;
-			if (body.media_ids) {
-				body.media_ids = (body.media_ids as string[]).map((p) =>
-					fromMastodonId(p),
-				);
-			}
-			const { sensitive } = body;
-			body.sensitive =
-				typeof sensitive === "string" ? sensitive === "true" : sensitive;
-
-			if (body.poll) {
-				if (
-					body.poll.expires_in != null &&
-					typeof body.poll.expires_in === "string"
-				)
-					body.poll.expires_in = parseInt(body.poll.expires_in);
-				if (
-					body.poll.multiple != null &&
-					typeof body.poll.multiple === "string"
-				)
-					body.poll.multiple = body.poll.multiple == "true";
-				if (
-					body.poll.hide_totals != null &&
-					typeof body.poll.hide_totals === "string"
-				)
-					body.poll.hide_totals = body.poll.hide_totals == "true";
-			}
-
-			const data = await client.postStatus(text, body);
-			ctx.body = convertStatus(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
 		}
+
+		const request = NoteHelpers.normalizeComposeOptions(ctx.request.body);
+		const status = await NoteHelpers.createNote(request, ctx).then(
+			async (p) => {
+				if (!request.scheduled_at) return NoteConverter.encode(p, ctx);
+				const note = await NoteHelpers.getScheduledNoteOr404(p.id, ctx);
+				const result = await NoteConverter.encodeScheduledNote(note, ctx);
+				result.params.idempotency = key ?? result.params.idempotency;
+				return result;
+			},
+		);
+		ctx.body = status;
+
+		if (key !== null && "text" in status)
+			NoteHelpers.postIdempotencyCache.set(key, { status });
 	});
-	router.get<{ Params: { id: string } }>("/v1/statuses/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getStatus(fromMastodonId(ctx.params.id));
-			ctx.body = convertStatus(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = ctx.status == 404 ? 404 : 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.delete<{ Params: { id: string } }>("/v1/statuses/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.deleteStatus(fromMastodonId(ctx.params.id));
-			ctx.body = data.data;
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	interface IReaction {
-		id: string;
-		createdAt: string;
-		user: MisskeyEntity.User;
-		type: string;
-	}
+
+	router.put(
+		"/v1/statuses/:id",
+		auth(true, ["write:statuses"]),
+		async (ctx) => {
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const request = NoteHelpers.normalizeEditOptions(ctx.request.body);
+			ctx.body = await NoteHelpers.editNote(request, note, ctx).then((p) =>
+				NoteConverter.encode(p, ctx),
+			);
+		},
+	);
+
+	router.get<{ Params: { id: string } }>(
+		"/v1/statuses/:id",
+		auth(false, ["read:statuses"]),
+		filterContext("thread"),
+		async (ctx) => {
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteConverter.encode(note, ctx);
+		},
+	);
+
+	router.delete<{ Params: { id: string } }>(
+		"/v1/statuses/:id",
+		auth(true, ["write:statuses"]),
+		async (ctx) => {
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			ctx.body = await NoteHelpers.deleteNote(note, ctx);
+		},
+	);
+
 	router.get<{ Params: { id: string } }>(
 		"/v1/statuses/:id/context",
+		auth(false, ["read:statuses"]),
+		filterContext("thread"),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const id = fromMastodonId(ctx.params.id);
-				const data = await client.getStatusContext(
-					id,
-					convertTimelinesArgsId(limitToInt(ctx.query as any)),
-				);
+			// FIXME: determine final limits within helper functions instead of here
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const ancestors = await NoteHelpers.getNoteAncestors(
+				note,
+				ctx.user ? 4096 : 60,
+				ctx,
+			).then((n) => NoteConverter.encodeMany(n, ctx));
+			const descendants = await NoteHelpers.getNoteDescendants(
+				note,
+				ctx.user ? 4096 : 40,
+				ctx.user ? 4096 : 20,
+				ctx,
+			).then((n) => NoteConverter.encodeMany(n, ctx));
 
-				data.data.ancestors = data.data.ancestors.map((status) =>
-					convertStatus(status),
-				);
-				data.data.descendants = data.data.descendants.map((status) =>
-					convertStatus(status),
-				);
-				ctx.body = data.data;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			ctx.body = {
+				ancestors,
+				descendants,
+			};
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/statuses/:id/history",
+		auth(false, ["read:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getStatusHistory(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			ctx.body = await NoteHelpers.getNoteEditHistory(note, ctx);
+		},
+	);
+	router.get<{ Params: { id: string } }>(
+		"/v1/statuses/:id/source",
+		auth(true, ["read:statuses"]),
+		async (ctx) => {
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			ctx.body = NoteHelpers.getNoteSource(note);
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/statuses/:id/reblogged_by",
+		auth(false, ["read:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getStatusRebloggedBy(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+			const res = await NoteHelpers.getNoteRebloggedBy(
+				note,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await UserConverter.encodeMany(res, ctx);
 		},
 	);
 	router.get<{ Params: { id: string } }>(
 		"/v1/statuses/:id/favourited_by",
+		auth(false, ["read:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getStatusFavouritedBy(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const args = normalizeUrlQuery(limitToInt(ctx.query as any));
+			const res = await NoteHelpers.getNoteFavoritedBy(
+				note,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await UserConverter.encodeMany(res, ctx);
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/favourite",
+		auth(true, ["write:favourites"]),
 		async (ctx) => {
-			const meta = await fetchMeta(true);
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			const react = meta.defaultReaction;
-			try {
-				const a = (await client.createEmojiReaction(
-					fromMastodonId(ctx.params.id),
-					react,
-				)) as any;
-				//const data = await client.favouriteStatus(ctx.params.id) as any;
-				ctx.body = convertStatus(a.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const reaction = await NoteHelpers.getDefaultReaction();
+
+			ctx.body = await NoteHelpers.reactToNote(note, reaction, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.favourited = true;
+					return p;
+				});
 		},
 	);
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/unfavourite",
+		auth(true, ["write:favourites"]),
 		async (ctx) => {
-			const meta = await fetchMeta(true);
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			const react = meta.defaultReaction;
-			try {
-				const data = await client.deleteEmojiReaction(
-					fromMastodonId(ctx.params.id),
-					react,
-				);
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.removeReactFromNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.favourited = false;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/reblog",
+		auth(true, ["write:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.reblogStatus(fromMastodonId(ctx.params.id));
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.reblogNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.reblogged = true;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/unreblog",
+		auth(true, ["write:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unreblogStatus(fromMastodonId(ctx.params.id));
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.unreblogNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.reblogged = false;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/bookmark",
+		auth(true, ["write:bookmarks"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.bookmarkStatus(fromMastodonId(ctx.params.id));
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.bookmarkNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.bookmarked = true;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/unbookmark",
+		auth(true, ["write:bookmarks"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unbookmarkStatus(
-					fromMastodonId(ctx.params.id),
-				);
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.unbookmarkNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.bookmarked = false;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/pin",
+		auth(true, ["write:accounts"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.pinStatus(fromMastodonId(ctx.params.id));
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.pinNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.pinned = true;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string } }>(
 		"/v1/statuses/:id/unpin",
+		auth(true, ["write:accounts"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unpinStatus(fromMastodonId(ctx.params.id));
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.unpinNote(note, ctx)
+				.then((p) => NoteConverter.encode(p, ctx))
+				.then((p) => {
+					p.pinned = false;
+					return p;
+				});
 		},
 	);
 
 	router.post<{ Params: { id: string; name: string } }>(
 		"/v1/statuses/:id/react/:name",
+		auth(true, ["write:favourites"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.reactStatus(
-					fromMastodonId(ctx.params.id),
-					ctx.params.name,
-				);
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.reactToNote(note, ctx.params.name, ctx).then(
+				(p) => NoteConverter.encode(p, ctx),
+			);
 		},
 	);
 
 	router.post<{ Params: { id: string; name: string } }>(
 		"/v1/statuses/:id/unreact/:name",
+		auth(true, ["write:favourites"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.unreactStatus(
-					fromMastodonId(ctx.params.id),
-					ctx.params.name,
-				);
-				ctx.body = convertStatus(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			ctx.body = await NoteHelpers.removeReactFromNote(note, ctx).then((p) =>
+				NoteConverter.encode(p, ctx),
+			);
+		},
+	);
+
+	router.post<{ Params: { id: string } }>(
+		"/v1/statuses/:id/translate",
+		auth(true, ["read:statuses"]),
+		async (ctx) => {
+			const targetLang = ctx.request.body.lang;
+			if (typeof targetLang !== "string")
+				throw new MastoApiError(400, "Missing lang parameter");
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			const encodedNote = await NoteConverter.encode(note, ctx);
+			ctx.body = await NoteHelpers.translateNote(encodedNote, targetLang, ctx);
+		},
+	);
+
+	router.get<{ Params: { id: string } }>(
+		"/v1/polls/:id",
+		auth(false, ["read:statuses"]),
+		async (ctx) => {
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+			ctx.body = await PollHelpers.getPoll(note, ctx);
 		},
 	);
 
-	router.get<{ Params: { id: string } }>("/v1/media/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getMedia(fromMastodonId(ctx.params.id));
-			ctx.body = convertAttachment(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.put<{ Params: { id: string } }>("/v1/media/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.updateMedia(
-				fromMastodonId(ctx.params.id),
-				ctx.request.body as any,
-			);
-			ctx.body = convertAttachment(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get<{ Params: { id: string } }>("/v1/polls/:id", async (ctx) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getPoll(fromMastodonId(ctx.params.id));
-			ctx.body = convertPoll(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
 	router.post<{ Params: { id: string } }>(
 		"/v1/polls/:id/votes",
+		auth(true, ["write:statuses"]),
 		async (ctx) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.votePoll(
-					fromMastodonId(ctx.params.id),
-					(ctx.request.body as any).choices,
-				);
-				ctx.body = convertPoll(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
+
+			const body: any = ctx.request.body;
+			const choices = toArray(body.choices ?? []).map((p) =>
+				Number.parseInt(p),
+			);
+			if (choices.length < 1)
+				throw new MastoApiError(400, "Must vote for at least one option");
+
+			ctx.body = await PollHelpers.voteInPoll(choices, note, ctx);
+		},
+	);
+
+	router.get(
+		"/v1/scheduled_statuses",
+		auth(true, ["read:statuses"]),
+		async (ctx) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query));
+
+			const res = await NoteHelpers.getScheduledNotes(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
+			);
+			ctx.body = await NoteConverter.encodeManyScheduledNotes(res, ctx);
+		},
+	);
+
+	router.get<{ Params: { id: string } }>(
+		"/v1/scheduled_statuses/:id",
+		auth(true, ["read:statuses"]),
+		async (ctx) => {
+			const note = await NoteHelpers.getScheduledNoteOr404(ctx.params.id, ctx);
+			ctx.body = await NoteConverter.encodeScheduledNote(note, ctx);
+		},
+	);
+
+	// Reeschedule a post to a new time
+	router.put<{ Params: { id: string } }>(
+		"/v1/scheduled_statuses/:id",
+		auth(true, ["write:statuses"]),
+		async (ctx) => {
+			const scheduledAt = new Date(Date.parse(ctx.request.body.scheduled_at));
+			// FIXME: Implement, see https://firefish.dev/firefish/firefish/-/issues/10903
+			throw new MastoApiError(501, "Not implemented");
+		},
+	);
+
+	router.delete<{ Params: { id: string } }>(
+		"/v1/scheduled_statuses/:id",
+		auth(true, ["write:statuses"]),
+		async (ctx) => {
+			// FIXME: Implement, see https://firefish.dev/firefish/firefish/-/issues/10903
+			throw new MastoApiError(501, "Not implemented");
 		},
 	);
 }

packages/backend/src/server/api/mastodon/endpoints/streaming.ts

@@ -0,0 +1,7 @@
+import type Router from "@koa/router";
+
+export function setupEndpointsStreaming(router: Router): void {
+	router.get("/v1/streaming/health", async (ctx) => {
+		ctx.body = "OK";
+	});
+}

packages/backend/src/server/api/mastodon/endpoints/timeline.ts

@@ -1,26 +1,28 @@
 import type Router from "@koa/router";
-import { getClient } from "../ApiMastodonCompatibleService.js";
 import type { ParsedUrlQuery } from "node:querystring";
-import {
-	convertAccount,
-	convertConversation,
-	convertList,
-	convertStatus,
-} from "../converters.js";
-import { fromMastodonId } from "backend-rs";
-import { apiLogger } from "@/server/api/logger.js";
-import { inspect } from "node:util";
+import { TimelineHelpers } from "@/server/api/mastodon/helpers/timeline.js";
+import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
+import { UserLists } from "@/models/index.js";
+import { auth } from "@/server/api/mastodon/middleware/auth.js";
+import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
+import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
 
-export function limitToInt(q: ParsedUrlQuery) {
-	let object: any = q;
+// TODO: Move helper functions to a helper class
+export function limitToInt(q: ParsedUrlQuery, additional: string[] = []) {
+	const object: any = q;
 	if (q.limit)
-		if (typeof q.limit === "string") object.limit = parseInt(q.limit, 10);
+		if (typeof q.limit === "string")
+			object.limit = Number.parseInt(q.limit, 10);
 	if (q.offset)
-		if (typeof q.offset === "string") object.offset = parseInt(q.offset, 10);
+		if (typeof q.offset === "string")
+			object.offset = Number.parseInt(q.offset, 10);
+	for (const key of additional)
+		if (typeof q[key] === "string")
+			object[key] = Number.parseInt(<string>q[key], 10);
 	return object;
 }
 
-export function argsToBools(q: ParsedUrlQuery) {
+export function argsToBools(q: ParsedUrlQuery, additional: string[] = []) {
 	// Values taken from https://docs.joinmastodon.org/client/intro/#boolean
 	const toBoolean = (value: string) =>
 		!["0", "f", "F", "false", "FALSE", "off", "OFF"].includes(value);
@@ -29,273 +31,158 @@ export function argsToBools(q: ParsedUrlQuery) {
 	// - https://docs.joinmastodon.org/methods/accounts/#statuses
 	// - https://docs.joinmastodon.org/methods/timelines/#public
 	// - https://docs.joinmastodon.org/methods/timelines/#tag
-	let object: any = q;
-	if (q.only_media)
-		if (typeof q.only_media === "string")
-			object.only_media = toBoolean(q.only_media);
-	if (q.exclude_replies)
-		if (typeof q.exclude_replies === "string")
-			object.exclude_replies = toBoolean(q.exclude_replies);
-	if (q.exclude_reblogs)
-		if (typeof q.exclude_reblogs === "string")
-			object.exclude_reblogs = toBoolean(q.exclude_reblogs);
-	if (q.pinned)
-		if (typeof q.pinned === "string") object.pinned = toBoolean(q.pinned);
-	if (q.local)
-		if (typeof q.local === "string") object.local = toBoolean(q.local);
-	return q;
+	const keys = [
+		"only_media",
+		"exclude_replies",
+		"exclude_reblogs",
+		"pinned",
+		"local",
+		"remote",
+	].concat(additional);
+	const object: any = q;
+
+	for (const key of keys)
+		if (q[key] && typeof q[key] === "string")
+			object[key] = toBoolean(<string>q[key]);
+
+	return object;
 }
 
-export function convertTimelinesArgsId(q: ParsedUrlQuery) {
-	if (typeof q.min_id === "string") q.min_id = fromMastodonId(q.min_id);
-	if (typeof q.max_id === "string") q.max_id = fromMastodonId(q.max_id);
-	if (typeof q.since_id === "string") q.since_id = fromMastodonId(q.since_id);
-	return q;
+export function normalizeUrlQuery(
+	q: ParsedUrlQuery,
+	arrayKeys: string[] = [],
+): any {
+	const dict: any = {};
+
+	for (const k in q) {
+		if (arrayKeys.includes(k)) dict[k] = Array.isArray(q[k]) ? q[k] : [q[k]];
+		else dict[k] = Array.isArray(q[k]) ? q[k]?.at(-1) : q[k];
+	}
+
+	return dict;
 }
 
-export function apiTimelineMastodon(router: Router): void {
-	router.get("/v1/timelines/public", async (ctx, reply) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const query: any = ctx.query;
-			const data =
-				query.local === "true"
-					? await client.getLocalTimeline(
-							convertTimelinesArgsId(argsToBools(limitToInt(query))),
-						)
-					: await client.getPublicTimeline(
-							convertTimelinesArgsId(argsToBools(limitToInt(query))),
-						);
-			ctx.body = data.data.map((status) => convertStatus(status));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+export function setupEndpointsTimeline(router: Router): void {
+	router.get(
+		"/v1/timelines/public",
+		auth(true, ["read:statuses"]),
+		filterContext("public"),
+		async (ctx, reply) => {
+			const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)));
+			const res = await TimelineHelpers.getPublicTimeline(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				args.only_media,
+				args.local,
+				args.remote,
+				ctx,
+			);
+			ctx.body = await NoteConverter.encodeMany(res, ctx);
+		},
+	);
+
 	router.get<{ Params: { hashtag: string } }>(
 		"/v1/timelines/tag/:hashtag",
+		auth(false, ["read:statuses"]),
+		filterContext("public"),
 		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getTagTimeline(
-					ctx.params.hashtag,
-					convertTimelinesArgsId(argsToBools(limitToInt(ctx.query))),
-				);
-				ctx.body = data.data.map((status) => convertStatus(status));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const tag = (ctx.params.hashtag ?? "").trim().toLowerCase();
+			const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)), [
+				"any[]",
+				"all[]",
+				"none[]",
+			]);
+			const res = await TimelineHelpers.getTagTimeline(
+				tag,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				args["any[]"] ?? [],
+				args["all[]"] ?? [],
+				args["none[]"] ?? [],
+				args.only_media,
+				args.local,
+				args.remote,
+				ctx,
+			);
+			ctx.body = await NoteConverter.encodeMany(res, ctx);
 		},
 	);
-	router.get("/v1/timelines/home", async (ctx, reply) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getHomeTimeline(
-				convertTimelinesArgsId(limitToInt(ctx.query)),
+
+	router.get(
+		"/v1/timelines/home",
+		auth(true, ["read:statuses"]),
+		filterContext("home"),
+		async (ctx, reply) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query));
+			const res = await TimelineHelpers.getHomeTimeline(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
 			);
-			ctx.body = data.data.map((status) => convertStatus(status));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
+			ctx.body = await NoteConverter.encodeMany(res, ctx);
+		},
+	);
+
 	router.get<{ Params: { listId: string } }>(
 		"/v1/timelines/list/:listId",
+		auth(true, ["read:lists"]),
+		filterContext("home"),
 		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getListTimeline(
-					fromMastodonId(ctx.params.listId),
-					convertTimelinesArgsId(limitToInt(ctx.query)),
-				);
-				ctx.body = data.data.map((status) => convertStatus(status));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
-		},
-	);
-	router.get("/v1/conversations", async (ctx, reply) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getConversationTimeline(
-				convertTimelinesArgsId(limitToInt(ctx.query)),
+			const list = await UserLists.findOneBy({
+				userId: ctx.user.id,
+				id: ctx.params.listId,
+			});
+			if (!list) throw new MastoApiError(404);
+
+			const args = normalizeUrlQuery(limitToInt(ctx.query));
+			const res = await TimelineHelpers.getListTimeline(
+				list,
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
 			);
-			ctx.body = data.data.map((conversation) =>
-				convertConversation(conversation),
+			ctx.body = await NoteConverter.encodeMany(res, ctx);
+		},
+	);
+
+	router.get(
+		"/v1/conversations",
+		auth(true, ["read:statuses"]),
+		async (ctx, reply) => {
+			const args = normalizeUrlQuery(limitToInt(ctx.query));
+			ctx.body = await TimelineHelpers.getConversations(
+				args.max_id,
+				args.since_id,
+				args.min_id,
+				args.limit,
+				ctx,
 			);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get("/v1/lists", async (ctx, reply) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.getLists();
-			ctx.body = data.data.map((list) => convertList(list));
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.get<{ Params: { id: string } }>(
-		"/v1/lists/:id",
-		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getList(fromMastodonId(ctx.params.id));
-				ctx.body = convertList(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
 		},
 	);
-	router.post("/v1/lists", async (ctx, reply) => {
-		const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-		const accessTokens = ctx.headers.authorization;
-		const client = getClient(BASE_URL, accessTokens);
-		try {
-			const data = await client.createList((ctx.request.body as any).title);
-			ctx.body = convertList(data.data);
-		} catch (e: any) {
-			apiLogger.error(inspect(e));
-			ctx.status = 401;
-			ctx.body = e.response.data;
-		}
-	});
-	router.put<{ Params: { id: string } }>(
-		"/v1/lists/:id",
+
+	router.get(
+		"/v1/markers",
+		auth(true, ["read:statuses"]),
 		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.updateList(
-					fromMastodonId(ctx.params.id),
-					(ctx.request.body as any).title,
-				);
-				ctx.body = convertList(data.data);
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const args = normalizeUrlQuery(ctx.query, ["timeline[]"]);
+			ctx.body = await TimelineHelpers.getMarkers(args["timeline[]"], ctx);
 		},
 	);
-	router.delete<{ Params: { id: string } }>(
-		"/v1/lists/:id",
+
+	router.post(
+		"/v1/markers",
+		auth(true, ["write:statuses"]),
 		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.deleteList(fromMastodonId(ctx.params.id));
-				ctx.body = data.data;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
-		},
-	);
-	router.get<{ Params: { id: string } }>(
-		"/v1/lists/:id/accounts",
-		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.getAccountsInList(
-					fromMastodonId(ctx.params.id),
-					convertTimelinesArgsId(ctx.query as any),
-				);
-				ctx.body = data.data.map((account) => convertAccount(account));
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
-		},
-	);
-	router.post<{ Params: { id: string } }>(
-		"/v1/lists/:id/accounts",
-		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.addAccountsToList(
-					fromMastodonId(ctx.params.id),
-					(ctx.query.account_ids as string[]).map((id) => fromMastodonId(id)),
-				);
-				ctx.body = data.data;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
-		},
-	);
-	router.delete<{ Params: { id: string } }>(
-		"/v1/lists/:id/accounts",
-		async (ctx, reply) => {
-			const BASE_URL = `${ctx.protocol}://${ctx.hostname}`;
-			const accessTokens = ctx.headers.authorization;
-			const client = getClient(BASE_URL, accessTokens);
-			try {
-				const data = await client.deleteAccountsFromList(
-					fromMastodonId(ctx.params.id),
-					(ctx.query.account_ids as string[]).map((id) => fromMastodonId(id)),
-				);
-				ctx.body = data.data;
-			} catch (e: any) {
-				apiLogger.error(inspect(e));
-				ctx.status = 401;
-				ctx.body = e.response.data;
-			}
+			const body = ctx.request.body;
+			ctx.body = await TimelineHelpers.setMarkers(body, ctx);
 		},
 	);
 }
-function escapeHTML(str: string) {
-	if (!str) {
-		return "";
-	}
-	return str
-		.replace(/&/g, "&amp;")
-		.replace(/</g, "&lt;")
-		.replace(/>/g, "&gt;")
-		.replace(/"/g, "&quot;")
-		.replace(/'/g, "&#039;");
-}
-function nl2br(str: string) {
-	if (!str) {
-		return "";
-	}
-	str = str.replace(/\r\n/g, "<br />");
-	str = str.replace(/(\n|\r)/g, "<br />");
-	return str;
-}

packages/backend/src/server/api/mastodon/entities/account.ts

@@ -1,11 +1,12 @@
 /// <reference path="emoji.ts" />
 /// <reference path="source.ts" />
 /// <reference path="field.ts" />
-namespace Entity {
+namespace MastodonEntity {
 	export type Account = {
 		id: string;
 		username: string;
 		acct: string;
+		fqn: string;
 		display_name: string;
 		locked: boolean;
 		created_at: string;
@@ -22,6 +23,18 @@ namespace Entity {
 		moved: Account | null;
 		fields: Array<Field>;
 		bot: boolean | null;
+		discoverable: boolean;
 		source?: Source;
 	};
+
+	export type MutedAccount =
+		| Account
+		| {
+				mute_expires_at: string | null;
+		  };
+
+	export type SuggestedAccount = {
+		source: "staff" | "past_interactions" | "global";
+		account: Account;
+	};
 }

packages/backend/src/server/api/mastodon/entities/activity.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Activity = {
 		week: string;
 		statuses: string;

packages/backend/src/server/api/mastodon/entities/announcement.ts

@@ -2,7 +2,7 @@
 /// <reference path="emoji.ts" />
 /// <reference path="reaction.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Announcement = {
 		id: string;
 		content: string;

packages/backend/src/server/api/mastodon/entities/application.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Application = {
 		name: string;
 		website?: string | null;

packages/backend/src/server/api/mastodon/entities/async_attachment.ts

@@ -1,5 +1,5 @@
 /// <reference path="attachment.ts" />
-namespace Entity {
+namespace MastodonEntity {
 	export type AsyncAttachment = {
 		id: string;
 		type: "unknown" | "image" | "gifv" | "video" | "audio";

packages/backend/src/server/api/mastodon/entities/attachment.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Sub = {
 		// For Image, Gifv, and Video
 		width?: number;

packages/backend/src/server/api/mastodon/entities/card.ts

@@ -0,0 +1,22 @@
+namespace MastodonEntity {
+	export type Card = {
+		url: string;
+		title: string;
+		description: string;
+		type: "link" | "photo" | "video" | "rich";
+		image: string | null;
+		author_name: string;
+		author_url: string;
+		provider_name: string;
+		provider_url: string;
+		html: string;
+		width: number;
+		height: number;
+		embed_url: string;
+		blurhash: string | null;
+		/** ISO 8901 date time string */
+		published_at: string | null;
+		image_description: string;
+		language: string | null;
+	};
+}

packages/backend/src/server/api/mastodon/entities/context.ts

@@ -1,6 +1,6 @@
 /// <reference path="status.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Context = {
 		ancestors: Array<Status>;
 		descendants: Array<Status>;

packages/backend/src/server/api/mastodon/entities/conversation.ts

@@ -1,7 +1,7 @@
 /// <reference path="account.ts" />
 /// <reference path="status.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Conversation = {
 		id: string;
 		accounts: Array<Account>;

packages/backend/src/server/api/mastodon/entities/emoji.ts

@@ -1,9 +1,9 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Emoji = {
 		shortcode: string;
 		static_url: string;
 		url: string;
 		visible_in_picker: boolean;
-		category: string;
+		category: string | undefined;
 	};
 }

packages/backend/src/server/api/mastodon/entities/featured_tag.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type FeaturedTag = {
 		id: string;
 		name: string;

packages/backend/src/server/api/mastodon/entities/field.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Field = {
 		name: string;
 		value: string;

packages/backend/src/server/api/mastodon/entities/files.ts

@@ -0,0 +1,5 @@
+import type { MastoContext } from "..";
+
+export type Files = MastoContext["request"]["files"];
+type FileOrFileArr = Exclude<Files, undefined>["file"];
+export type File = Exclude<FileOrFileArr, FileOrFileArr[]>;

packages/backend/src/server/api/mastodon/entities/filter.ts

@@ -0,0 +1,18 @@
+namespace MastodonEntity {
+	export type Filter = {
+		id: string;
+		title: string;
+		context: Array<FilterContext>;
+		expires_at: string | null;
+		filter_action: "warn" | "hide";
+		keywords: FilterKeyword[];
+		statuses: FilterStatus[];
+	};
+
+	export type FilterContext =
+		| "home"
+		| "notifications"
+		| "public"
+		| "thread"
+		| "account";
+}

packages/backend/src/server/api/mastodon/entities/filter_keyword.ts

@@ -0,0 +1,7 @@
+namespace MastodonEntity {
+	export type FilterKeyword = {
+		id: string;
+		keyword: string;
+		whole_word: boolean;
+	};
+}

packages/backend/src/server/api/mastodon/entities/filter_result.ts

@@ -0,0 +1,7 @@
+namespace MastodonEntity {
+	export type FilterResult = {
+		filter: Filter;
+		keyword_matches?: string[];
+		status_matches?: string[];
+	};
+}

packages/backend/src/server/api/mastodon/entities/filter_status.ts

@@ -0,0 +1,6 @@
+namespace MastodonEntity {
+	export type FilterStatus = {
+		id: string;
+		status_id: string;
+	};
+}

packages/backend/src/server/api/mastodon/entities/history.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type History = {
 		day: string;
 		uses: number;

packages/backend/src/server/api/mastodon/entities/identity_proof.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type IdentityProof = {
 		provider: string;
 		provider_username: string;

packages/backend/src/server/api/mastodon/entities/instance.ts

@@ -0,0 +1,109 @@
+/// <reference path="account.ts" />
+/// <reference path="urls.ts" />
+/// <reference path="stats.ts" />
+
+namespace MastodonEntity {
+	export type Instance = {
+		uri: string;
+		title: string;
+		description: string;
+		email: string;
+		version: string;
+		thumbnail: string | null;
+		urls: URLs;
+		stats: Stats;
+		languages: Array<string>;
+		contact_account: Account | null;
+		max_toot_chars?: number;
+		registrations?: boolean;
+		configuration?: {
+			statuses: {
+				max_characters: number;
+				max_media_attachments: number;
+				characters_reserved_per_url: number;
+			};
+			media_attachments: {
+				supported_mime_types: Array<string>;
+				image_size_limit: number;
+				image_matrix_limit: number;
+				video_size_limit: number;
+				video_frame_limit: number;
+				video_matrix_limit: number;
+			};
+			polls: {
+				max_options: number;
+				max_characters_per_option: number;
+				min_expiration: number;
+				max_expiration: number;
+			};
+		};
+	};
+
+	export type InstanceV2 = {
+		domain: string;
+		title: string;
+		version: string;
+		source_url: string;
+		description: string;
+		usage: {
+			users: {
+				active_month: number;
+			};
+		};
+		thumbnail: {
+			url: string;
+			blurhash?: string;
+			versions?: {
+				"@1x"?: string;
+				"@2x"?: string;
+			};
+		};
+		languages: string[];
+		configuration: {
+			urls: {
+				streaming: string;
+			};
+			vapid: {
+				public_key: string;
+			};
+			accounts: {
+				max_featured_tags: number;
+			};
+			statuses: {
+				max_characters: number;
+				max_media_attachments: number;
+				characters_reserved_per_url: number;
+			};
+			media_attachments: {
+				supported_mime_types: string[];
+				image_size_limit: number;
+				image_matrix_limit: number;
+				video_size_limit: number;
+				video_frame_rate_limit: number;
+				video_matrix_limit: number;
+			};
+			polls: {
+				max_options: number;
+				max_characters_per_option: number;
+				min_expiration: number;
+				max_expiration: number;
+			};
+			translation: {
+				enabled: boolean;
+			};
+		};
+		registrations: {
+			enabled: boolean;
+			approval_required: boolean;
+			message: string | null;
+		};
+		contact: {
+			email: string;
+			account: Account | null;
+		};
+		rules: {
+			id: string;
+			text: string;
+		}[];
+	};
+}

packages/backend/src/server/api/mastodon/entities/list.ts

@@ -1,6 +1,7 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type List = {
 		id: string;
 		title: string;
+		exclusive: boolean;
 	};
 }

packages/backend/src/server/api/mastodon/entities/marker.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Marker = {
 		home?: {
 			last_read_id: string;

packages/backend/src/server/api/mastodon/entities/mention.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Mention = {
 		id: string;
 		username: string;

packages/backend/src/server/api/mastodon/entities/notification.ts

@@ -1,7 +1,7 @@
 /// <reference path="account.ts" />
 /// <reference path="status.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Notification = {
 		account: Account;
 		created_at: string;
@@ -11,5 +11,15 @@ namespace Entity {
 		type: NotificationType;
 	};
 
-	export type NotificationType = string;
+	export type NotificationType =
+		| "mention"
+		| "status"
+		| "reblog"
+		| "follow"
+		| "follow_request"
+		| "favourite"
+		| "poll"
+		| "update"
+		| "admin.sign_up"
+		| "admin.report";
 }

packages/backend/src/server/api/mastodon/entities/oauth/oauth.ts

@@ -3,11 +3,12 @@
  * Response data when oauth request.
  **/
 namespace OAuth {
-	export type AppDataFromServer = {
+	export type Application = {
 		id: string;
 		name: string;
 		website: string | null;
 		redirect_uri: string;
+		vapid_key: string | undefined;
 		client_id: string;
 		client_secret: string;
 	};
@@ -21,49 +22,9 @@ namespace OAuth {
 		refresh_token: string | null;
 	};
 
-	export class AppData {
-		public url: string | null;
-		public session_token: string | null;
-		constructor(
-			public id: string,
-			public name: string,
-			public website: string | null,
-			public redirect_uri: string,
-			public client_id: string,
-			public client_secret: string,
-		) {
-			this.url = null;
-			this.session_token = null;
-		}
-
-		/**
-		 * Serialize raw application data from server
-		 * @param raw from server
-		 */
-		static from(raw: AppDataFromServer) {
-			return new this(
-				raw.id,
-				raw.name,
-				raw.website,
-				raw.redirect_uri,
-				raw.client_id,
-				raw.client_secret,
-			);
-		}
-
-		get redirectUri() {
-			return this.redirect_uri;
-		}
-		get clientId() {
-			return this.client_id;
-		}
-		get clientSecret() {
-			return this.client_secret;
-		}
-	}
-
 	export class TokenData {
 		public _scope: string;
+
 		constructor(
 			public access_token: string,
 			public token_type: string,
@@ -96,21 +57,26 @@ namespace OAuth {
 		get accessToken() {
 			return this.access_token;
 		}
+
 		get tokenType() {
 			return this.token_type;
 		}
+
 		get scope() {
 			return this._scope;
 		}
+
 		/**
 		 * Application ID
 		 */
 		get createdAt() {
 			return this.created_at;
 		}
+
 		get expiresIn() {
 			return this.expires_in;
 		}
+
 		/**
 		 * OAuth Refresh Token
 		 */

packages/backend/src/server/api/mastodon/entities/poll.ts

@@ -1,6 +1,6 @@
 /// <reference path="poll_option.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Poll = {
 		id: string;
 		expires_at: string | null;
@@ -10,5 +10,6 @@ namespace Entity {
 		options: Array<PollOption>;
 		voted: boolean;
 		own_votes: Array<number>;
+		emojis: Array<MastodonEntity.Emoji>;
 	};
 }

packages/backend/src/server/api/mastodon/entities/poll_option.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type PollOption = {
 		title: string;
 		votes_count: number | null;

packages/backend/src/server/api/mastodon/entities/preferences.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Preferences = {
 		"posting:default:visibility": "public" | "unlisted" | "private" | "direct";
 		"posting:default:sensitive": boolean;

packages/backend/src/server/api/mastodon/entities/push_subscription.ts

@@ -0,0 +1,27 @@
+namespace MastodonEntity {
+	export type Alerts = {
+		follow: boolean;
+		favourite: boolean;
+		mention: boolean;
+		reblog: boolean;
+		poll: boolean;
+		status: boolean;
+	};
+
+	export type PushSubscription = {
+		id: string;
+		endpoint: string;
+		server_key: string;
+		alerts: Alerts;
+	};
+
+	export type NotificationPayload = {
+		access_token: string;
+		preferred_locale: string;
+		notification_id: string;
+		notification_type: string;
+		icon?: string;
+		title?: string;
+		body?: string;
+	};
+}

packages/backend/src/server/api/mastodon/entities/reaction.ts

@@ -1,6 +1,6 @@
 /// <reference path="account.ts" />
 
-namespace Entity {
+namespace MastodonEntity {
 	export type Reaction = {
 		count: number;
 		me: boolean;

packages/backend/src/server/api/mastodon/entities/relationship.ts

@@ -1,9 +1,8 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Relationship = {
 		id: string;
 		following: boolean;
 		followed_by: boolean;
-		delivery_following?: boolean;
 		blocking: boolean;
 		blocked_by: boolean;
 		muting: boolean;
@@ -13,5 +12,6 @@ namespace Entity {
 		showing_reblogs: boolean;
 		endorsed: boolean;
 		notifying: boolean;
+		note: string;
 	};
 }

packages/backend/src/server/api/mastodon/entities/report.ts

@@ -1,4 +1,4 @@
-namespace Entity {
+namespace MastodonEntity {
 	export type Report = {
 		id: string;
 		action_taken: string;

packages/backend/src/server/api/mastodon/entities/results.ts

@@ -2,8 +2,8 @@
 /// <reference path="status.ts" />
 /// <reference path="tag.ts" />
 
-namespace Entity {
-	export type Results = {
+namespace MastodonEntity {
+	export type Search = {
 		accounts: Array<Account>;
 		statuses: Array<Status>;
 		hashtags: Array<Tag>;