From f5cb9f5fa0f8416548e4661234eff5128aa37e33 Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 14 Nov 2024 21:15:59 -0700 Subject: [PATCH 01/21] rewrite initial --- .gitignore | 5 +- .../migrations/20241113204238_/migration.sql | 8 + .../migrations/20241113212417_/migration.sql | 25 ++ .../migrations/20241114021022_/migration.sql | 34 ++ .../migrations/20241115021410_/migration.sql | 2 + .../migrations/20241115022132_/migration.sql | 35 ++ prisma/schema.prisma | 19 +- src/discord.ts | 389 +++++++++++------- src/lib/glue.ts | 32 ++ src/lib/matrix.ts | 53 +-- src/matrix.ts | 257 ++++++------ src/types/env.ts | 31 ++ src/webserver.ts | 120 +++++- 13 files changed, 696 insertions(+), 314 deletions(-) create mode 100644 prisma/migrations/20241113204238_/migration.sql create mode 100644 prisma/migrations/20241113212417_/migration.sql create mode 100644 prisma/migrations/20241114021022_/migration.sql create mode 100644 prisma/migrations/20241115021410_/migration.sql create mode 100644 prisma/migrations/20241115022132_/migration.sql create mode 100644 src/lib/glue.ts diff --git a/.gitignore b/.gitignore index a05b628..e0dde9c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ node_modules .env prisma/dev.db -dist \ No newline at end of file +dist + +*.crt +*.key \ No newline at end of file diff --git a/prisma/migrations/20241113204238_/migration.sql b/prisma/migrations/20241113204238_/migration.sql new file mode 100644 index 0000000..50b7032 --- /dev/null +++ b/prisma/migrations/20241113204238_/migration.sql @@ -0,0 +1,8 @@ +-- CreateTable +CREATE TABLE "Bridged" ( + "discord_guild_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "matrix_room_id" TEXT NOT NULL, + + PRIMARY KEY ("discord_guild_id", "discord_channel_id", "matrix_room_id") +); diff --git a/prisma/migrations/20241113212417_/migration.sql b/prisma/migrations/20241113212417_/migration.sql new file mode 100644 index 0000000..3926317 --- /dev/null +++ b/prisma/migrations/20241113212417_/migration.sql @@ -0,0 +1,25 @@ +/* + Warnings: + + - The primary key for the `DiscordMessages` table will be changed. If it partially fails, the table could be left without primary key constraint. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_DiscordMessages" ( + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL, + "matrix_event_id" TEXT NOT NULL, + + PRIMARY KEY ("discord_id", "matrix_event_id") +); +INSERT INTO "new_DiscordMessages" ("discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id") SELECT "discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id" FROM "DiscordMessages"; +DROP TABLE "DiscordMessages"; +ALTER TABLE "new_DiscordMessages" RENAME TO "DiscordMessages"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; + +-- CreateIndex +CREATE INDEX "MatrixMessages_matrix_event_id_discord_id_idx" ON "MatrixMessages"("matrix_event_id", "discord_id"); diff --git a/prisma/migrations/20241114021022_/migration.sql b/prisma/migrations/20241114021022_/migration.sql new file mode 100644 index 0000000..c3c126b --- /dev/null +++ b/prisma/migrations/20241114021022_/migration.sql @@ -0,0 +1,34 @@ +/* + Warnings: + + - The primary key for the `DiscordMessages` table will be changed. If it partially fails, the table could be left without primary key constraint. + - The primary key for the `MatrixMessages` table will be changed. If it partially fails, the table could be left without primary key constraint. + - The required column `id` was added to the `DiscordMessages` table with a prisma-level default value. This is not possible if the table is not empty. Please add this column as optional, then populate it before making it required. + - The required column `id` was added to the `MatrixMessages` table with a prisma-level default value. This is not possible if the table is not empty. Please add this column as optional, then populate it before making it required. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_DiscordMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL, + "matrix_event_id" TEXT NOT NULL +); +INSERT INTO "new_DiscordMessages" ("discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id") SELECT "discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id" FROM "DiscordMessages"; +DROP TABLE "DiscordMessages"; +ALTER TABLE "new_DiscordMessages" RENAME TO "DiscordMessages"; +CREATE TABLE "new_MatrixMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "matrix_event_id" TEXT NOT NULL, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL +); +INSERT INTO "new_MatrixMessages" ("discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id") SELECT "discord_channel_id", "discord_guild_id", "discord_id", "matrix_event_id" FROM "MatrixMessages"; +DROP TABLE "MatrixMessages"; +ALTER TABLE "new_MatrixMessages" RENAME TO "MatrixMessages"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; diff --git a/prisma/migrations/20241115021410_/migration.sql b/prisma/migrations/20241115021410_/migration.sql new file mode 100644 index 0000000..f6ccda2 --- /dev/null +++ b/prisma/migrations/20241115021410_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "DiscordMessages" ADD COLUMN "attachment_id" TEXT; diff --git a/prisma/migrations/20241115022132_/migration.sql b/prisma/migrations/20241115022132_/migration.sql new file mode 100644 index 0000000..4badc08 --- /dev/null +++ b/prisma/migrations/20241115022132_/migration.sql @@ -0,0 +1,35 @@ +/* + Warnings: + + - Added the required column `matrix_room_id` to the `DiscordMessages` table without a default value. This is not possible if the table is not empty. + - Added the required column `matrix_room_id` to the `MatrixMessages` table without a default value. This is not possible if the table is not empty. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_DiscordMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL, + "attachment_id" TEXT, + "matrix_event_id" TEXT NOT NULL, + "matrix_room_id" TEXT NOT NULL +); +INSERT INTO "new_DiscordMessages" ("attachment_id", "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id") SELECT "attachment_id", "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id" FROM "DiscordMessages"; +DROP TABLE "DiscordMessages"; +ALTER TABLE "new_DiscordMessages" RENAME TO "DiscordMessages"; +CREATE TABLE "new_MatrixMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "matrix_event_id" TEXT NOT NULL, + "matrix_room_id" TEXT NOT NULL, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL +); +INSERT INTO "new_MatrixMessages" ("discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id") SELECT "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id" FROM "MatrixMessages"; +DROP TABLE "MatrixMessages"; +ALTER TABLE "new_MatrixMessages" RENAME TO "MatrixMessages"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 06514c7..fda87d7 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -10,22 +10,37 @@ datasource db { url = env("DATABASE_URL") } +model Bridged { + discord_guild_id String + discord_channel_id String + matrix_room_id String + + @@id([discord_guild_id, discord_channel_id, matrix_room_id]) +} + model DiscordWebhooks { channel_id String @id webhook_id String webhook_token String } +// discord -> matrix model DiscordMessages { - discord_id String @id + id String @id @default(uuid()) // one message may get represented as multiple + discord_id String discord_channel_id String discord_guild_id String + attachment_id String? matrix_event_id String + matrix_room_id String } +// matrix -> discord model MatrixMessages { - matrix_event_id String @id + id String @id @default(uuid()) // one message may get represented as multiple + matrix_event_id String + matrix_room_id String discord_id String discord_channel_id String diff --git a/src/discord.ts b/src/discord.ts index a94d7d6..e379c56 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -15,6 +15,7 @@ import { MessageCreateOptions, MessagePayload, PartialMessage, + Partials, Typing, User, Webhook, @@ -30,6 +31,7 @@ import { import { Matrix } from "./lib/matrix"; import { FriendlyError } from "./lib/utils"; import { Turndown } from "./lib/turndown"; +import { Glue } from "./lib/glue"; class Discord_ { client = new Client({ @@ -42,13 +44,14 @@ class Discord_ { GatewayIntentBits.GuildMessageTyping, GatewayIntentBits.GuildMessageReactions, ], + partials: [Partials.Message], }); constructor() { this.client.on("messageCreate", this.handleMessageCreate.bind(this)); this.client.on("messageUpdate", this.handleMessageUpdate.bind(this)); this.client.on("messageDelete", this.handleMessageDelete.bind(this)); - this.client.on("typingStart", this.handleTypingStart.bind(this)); + // this.client.on("typingStart", this.handleTypingStart.bind(this)); this.client.on("ready", () => { console.log(`Logged in as ${this.client.user?.tag}`); @@ -59,68 +62,125 @@ class Discord_ { this.client.login(token); } + async getAttachmentURL( + guild_id: string, + channel_id: string, + message_id: string, + attachment_id: string + ) { + const guild = await this.client.guilds.fetch(guild_id); + const channel = await guild.channels.fetch(channel_id); + if (!channel || channel.type !== ChannelType.GuildText) return; + const message = await channel.messages.fetch(message_id); + return message.attachments.get(attachment_id)?.url; + } + async handleMessageCreate(message: Message) { - if (message.author.id === this.client.user!.id) return; - if (message.webhookId) return; + if (message.author.id === this.client.user!.id) return; // ignore bot's own messages + if (message.webhookId) return; // if it's a webhook we don't care about it if (!message.guildId) return; // we only care about guild messages - const roomId = await Matrix.getRoomIDForDiscord( - message.guildId, - message.channelId - ); - if (!roomId) return; - // jank fix to prevent people from claiming aliases - // this room is just a bridge void - if (roomId === "!lqCdoAlvRDYMkAONOU:aftermath.gg") return; + const bridged = await Glue.getBridges({ + discord_guild: message.guildId, + discord_channel: message.channelId, + }); + if (!bridged.length) return; // no valid bridges var content = message.content; content = await this.replaceRoleMentions(message.guildId, content); + let matrixClient; try { - const matrixClient = await Matrix.for(`_discord_${message.author.id}`); + matrixClient = await Matrix.for(`_discord_${message.author.id}`); await matrixClient.syncProfile(message.author.displayName); - await Matrix.ensureUserInRoom(roomId, matrixClient.username); - const resp = await matrixClient.sendEvent( - roomId, - { - type: "m.room.message", - content: { - msgtype: "m.text", - body: - content + - (message.attachments.size > 0 - ? "\n" + - message.attachments - .map((attachment) => attachment.url) - .join("\n") - : ""), - "m.mentions": { - user_ids: Matrix.scanForMatrixMentions(message.content), - }, - }, - }, - message.id - ); + } catch (e) { + console.error("Failed to get bridged ghost user", e); + return; + } - if (resp.status === 200) { - await prisma.discordMessages.create({ - data: { - discord_id: message.id, - discord_channel_id: message.channelId, - discord_guild_id: message.guildId!, - matrix_event_id: resp.data.event_id, - }, - }); - } else { - await message.reply( - "Failed to send message: " + (resp.data.error || resp.data.errcode) + for (const room of bridged) { + try { + await Matrix.ensureUserInRoom( + room.matrix_room_id, + matrixClient.username ); - } - } catch (e) { - if (e instanceof FriendlyError) { - await message.reply("Error while bridging: " + e.message); - } else { - await message.reply("Unknown error while trying to bridge message"); + + await matrixClient + .sendEvent( + room.matrix_room_id, + { + type: "m.room.message", + content: { + msgtype: "m.text", + body: content, + "m.mentions": { + user_ids: Matrix.scanForMatrixMentions(message.content), + }, + }, + }, + [room.matrix_room_id, message.id].join(",") + ) + .then((result) => { + if (result.status === 200) { + return prisma.discordMessages.create({ + data: { + discord_guild_id: message.guildId!, + discord_channel_id: message.channelId, + discord_id: message.id, + matrix_event_id: result.data.event_id, + matrix_room_id: room.matrix_room_id, + }, + }); + } else { + console.error("Failed to send message", result); + } + }); + + for (const [id, attachment] of message.attachments) { + await matrixClient + .sendEvent( + room.matrix_room_id, + { + type: "m.room.message", + content: { + msgtype: "m.image", + body: "attachment.png", + url: `mxc://${process.env.HOST}/${Buffer.from([message.guildId, message.channelId, message.id, id].join(",")).toString("base64url")}`, + info: { + mimetype: attachment.contentType!, + // w: 128, + // h: 128, + }, + "m.mentions": { + user_ids: [], + }, + }, + }, + [room.matrix_room_id, message.id, id].join(",") + ) + .then((result) => { + if (result.status === 200) { + return prisma.discordMessages.create({ + data: { + discord_guild_id: message.guildId!, + discord_channel_id: message.channelId, + discord_id: message.id, + attachment_id: id, + matrix_event_id: result.data.event_id, + matrix_room_id: room.matrix_room_id, + }, + }); + } else { + console.error("Failed to send message", result); + } + }); + } + } catch (e) { + if (e instanceof FriendlyError) { + console.error("Failed to bridge to room [friendly]", room, e); + } else { + console.error("Failed to bridge to room", room, e); + } } } } @@ -133,128 +193,165 @@ class Discord_ { if (!newMessage.author || !newMessage.guildId || !oldMessage.guildId) return; - const db_message = await prisma.discordMessages.findFirst({ - where: { discord_id: newMessage.id }, + const messages = await prisma.discordMessages.findMany({ + where: { + discord_id: newMessage.id, + }, }); - if (!db_message) return; - const roomId = await Matrix.getRoomIDForDiscord( - newMessage.guildId, - newMessage.channelId - ); - if (!roomId) return; + if (!messages.length) return; - let content = newMessage.content; + const toDeleteAttachments = new Set(); + + for (const [id] of oldMessage.attachments) { + // loop through old message attachment IDs + // if the old ID isn't in the new message, it's been deleted + if (!newMessage.attachments.get(id)) { + toDeleteAttachments.add(id); + } + } const matrixClient = await this.matrix(newMessage.author); await matrixClient.syncProfile(newMessage.author.displayName); - await Matrix.ensureUserInRoom(roomId, matrixClient.username); - const resp = await matrixClient.sendEvent( - roomId, - { - type: "m.room.message", - content: { - msgtype: "m.text", - body: "* " + content, - "m.relates_to": { - rel_type: "m.replace", - event_id: db_message.matrix_event_id, - }, - "m.new_content": { + + for (const attachmentId of toDeleteAttachments) { + const dbMessages = messages.filter( + (m) => m.attachment_id === attachmentId + ); + + for (const dbMessage of dbMessages) { + try { + await Matrix.ensureUserInRoom( + dbMessage.matrix_room_id, + matrixClient.username + ); + await matrixClient.redactEvent( + dbMessage.matrix_room_id, + dbMessage.matrix_event_id, + dbMessage.id + "_" + Date.now() + ); + await prisma.discordMessages.delete({ + where: { + id: dbMessage.id, + }, + }); + } catch (e) { + console.log("Failed to delete event", dbMessage, e); + } + } + } + + let content = newMessage.content; + + const contentMessages = messages.filter((m) => !m.attachment_id); + for (const dbMessage of contentMessages) { + await Matrix.ensureUserInRoom( + dbMessage.matrix_room_id, + matrixClient.username + ); + const resp = await matrixClient.sendEvent( + dbMessage.matrix_room_id, + { + type: "m.room.message", + content: { msgtype: "m.text", - body: content!, - }, - "m.mentions": { - // we shouldn't mention anyone in a edit event - user_ids: [], + body: "* " + content, + "m.relates_to": { + rel_type: "m.replace", + event_id: dbMessage.matrix_event_id, + }, + "m.new_content": { + msgtype: "m.text", + body: content!, + }, + "m.mentions": { + // we shouldn't mention anyone in a edit event + user_ids: [], + }, }, }, - }, - newMessage.id + "_" + newMessage.editedTimestamp - ); - - if (resp.status === 200) { - await prisma.discordMessages.update({ - where: { discord_id: newMessage.id }, - data: { - discord_id: newMessage.id, - matrix_event_id: resp.data.event_id, - }, - }); - } else { - await newMessage.reply( - "Failed to send editted message: " + - (resp.data.error || resp.data.errcode) + [ + dbMessage.matrix_room_id, + newMessage.id, + newMessage.editedTimestamp, + ].join("_") ); } + + // if (resp.status === 200) { + // await prisma.discordMessages.update({ + // where: { discord_id: newMessage.id }, + // data: { + // discord_id: newMessage.id, + // matrix_event_id: resp.data.event_id, + // }, + // }); + // } else { + // await newMessage.reply( + // "Failed to send editted message: " + + // (resp.data.error || resp.data.errcode) + // ); + // } } async handleMessageDelete(message: Message | PartialMessage) { if (!message.author || !message.guildId) return; - const db_message = await prisma.discordMessages.findFirst({ + const discordMessages = await prisma.discordMessages.findMany({ where: { discord_id: message.id }, }); - - if (db_message) { - // this is a discord -> matrix message - - const roomId = await Matrix.getRoomIDForDiscord( - message.guildId, - message.channelId - ); - if (!roomId) return; - - const matrixClient = await this.matrix(message.author); - await Matrix.ensureUserInRoom(roomId, matrixClient.username); - const resp = await matrixClient.redactEvent( - roomId, - db_message.matrix_event_id, - message.id + "_" + Date.now() - ); - - if (resp.status === 200) { - await prisma.discordMessages.delete({ - where: { discord_id: message.id }, - }); - } else { - await message.channel.send( - "<@" + - message.author.id + - "> failed to send delete: " + - (resp.data.error || resp.data.errcode) + const user_matrixClient = await this.matrix(message.author); + const system_matrixClient = await Matrix.for("_discord_bot"); + + for (const dbMessage of discordMessages) { + try { + await Matrix.ensureUserInRoom( + dbMessage.matrix_room_id, + user_matrixClient.username ); - } - } else { - // this might be a matrix -> discord message - - const db2_message = await prisma.matrixMessages.findFirst({ - where: { discord_id: message.id }, - }); - - if (db2_message) { - // this *is* a matrix -> discord message - - await prisma.matrixMessages.delete({ + await user_matrixClient.redactEvent( + dbMessage.matrix_room_id, + dbMessage.matrix_event_id, + message.id + "_" + Date.now() + ); + await prisma.discordMessages.delete({ where: { - matrix_event_id: db2_message.matrix_event_id, + id: dbMessage.id, }, }); + } catch (e) { + console.error("Failed to delete message", dbMessage, e); + } + } - const roomId = await Matrix.getRoomIDForDiscord( - message.guildId, - message.channelId - ); - if (!roomId) return; + const matrixMessages = await prisma.matrixMessages.findMany({ + where: { discord_id: message.id }, + }); - const matrixClient = await Matrix.for("_discord_bot"); - await matrixClient.redactEvent( - roomId, - db2_message.matrix_event_id, + for (const dbMessage of matrixMessages) { + try { + await system_matrixClient.redactEvent( + dbMessage.matrix_room_id, + dbMessage.matrix_event_id, message.id + "_" + Date.now() ); + } catch (e) { + console.error("Failed to delete message", dbMessage, e); } } + + // if (resp.status === 200) { + // await prisma.discordMessages.delete({ + // where: { discord_id: message.id }, + // }); + // } else { + // await message.channel.send( + // "<@" + + // message.author.id + + // "> failed to send delete: " + + // (resp.data.error || resp.data.errcode) + // ); + // } } async handleTypingStart(typing: Typing) {} diff --git a/src/lib/glue.ts b/src/lib/glue.ts new file mode 100644 index 0000000..684d1b2 --- /dev/null +++ b/src/lib/glue.ts @@ -0,0 +1,32 @@ +import { prisma } from "./prisma"; + +export class Glue { + static async getBridges( + opts: + | { discord_guild: string; discord_channel: string } + | { matrix_room: string } + ) { + if ("discord_guild" in opts) { + const bridges = await prisma.bridged.findMany({ + where: { + discord_guild_id: opts.discord_guild, + discord_channel_id: opts.discord_channel, + }, + }); + + return bridges; + } + + if ("matrix_room" in opts) { + const bridges = await prisma.bridged.findMany({ + where: { + matrix_room_id: opts.matrix_room, + }, + }); + + return bridges; + } + + throw new Error("Missing required arguments"); + } +} diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index e456d34..081d98d 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -8,7 +8,12 @@ import { } from "../types/matrix"; import { FriendlyError } from "./utils"; -const { MATRIX_AS_TOKEN, MATRIX_HOMESERVER } = process.env; +const { MATRIX_AS_TOKEN } = process.env; +/** + * Homeserver friendly name (not direct access) + */ +const MATRIX_HOMESERVER = + process.env.MATRIX_HOMESERVER_HOST ?? process.env.MATRIX_HOMESERVER; interface MatrixEvents { joined: (user_id: IUsername) => void; @@ -54,14 +59,19 @@ export class Matrix { | { status: 400 | 403 | 429; data: { errcode: string; error?: string } } > { console.log("[->Matrix] " + method + " " + endpoint); - const req = await fetch(`https://${MATRIX_HOMESERVER}${endpoint}`, { - method, - headers: { - "Content-Type": "application/json", - Authorization: "Bearer " + MATRIX_AS_TOKEN, - }, - body: JSON.stringify(data), - }); + + let protocol = process.env.NODE_ENV === "development" ? "http" : "https"; + const req = await fetch( + `${protocol}://${process.env.MATRIX_HOMESERVER}${endpoint}`, + { + method, + headers: { + "Content-Type": "application/json", + Authorization: "Bearer " + MATRIX_AS_TOKEN, + }, + body: JSON.stringify(data), + } + ); const res = await req.json(); @@ -306,29 +316,6 @@ export class Matrix { * @see https://spec.matrix.org/v1.10/application-service-api/#using-sync-and-events */ sync() {} - /** - * Check if a Discord channel has a matching Matrix room - * - * TODO: this should be cached - * - * @param guild_id - * @param channel_id - * @returns - */ - static async getRoomIDForDiscord( - guild_id: string, - channel_id: string - ): Promise { - const req = await this.fetch<{ room_id: string }>( - `/_matrix/client/v3/directory/room/${encodeURIComponent(`#_discord_${guild_id}_${channel_id}:${MATRIX_HOMESERVER}`)}` - ); - - if (req.status === 200) { - return req.data.room_id; - } else { - return undefined; - } - } async joinRoom(roomIdOrAlias: string) { const req = await this.fetch( @@ -416,7 +403,7 @@ export class Matrix { return mxc.replace( "mxc://", - `https://${MATRIX_HOMESERVER}/_matrix/media/v3/download/` + `https://${process.env.MATRIX_HOMESERVER}/_matrix/media/v3/download/` ); } diff --git a/src/matrix.ts b/src/matrix.ts index ce140aa..4ab5bd0 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -1,4 +1,5 @@ import { Discord } from "./discord"; +import { Glue } from "./lib/glue"; import { Matrix } from "./lib/matrix"; import { prisma } from "./lib/prisma"; import { @@ -9,6 +10,12 @@ import { MatrixRoomRedaction, } from "./types/matrix"; +/** + * Homeserver friendly name (not direct access) + */ +const MATRIX_HOMESERVER = + process.env.MATRIX_HOMESERVER_HOST ?? process.env.MATRIX_HOMESERVER; + class MatrixHandler_ { private handledTransactionIDs: string[] = []; @@ -91,10 +98,7 @@ class MatrixHandler_ { if (event.content.membership === "invite") { // if someone is being invited, we only want to react when it's related to one of the virual accounts - if ( - event.state_key === - "@_discord_bot:" + process.env.MATRIX_HOMESERVER - ) { + if (event.state_key === "@_discord_bot:" + MATRIX_HOMESERVER) { const matrixClient = await Matrix.for("_discord_bot"); if (event.sender !== process.env.MATRIX_ADMIN) { @@ -109,13 +113,12 @@ class MatrixHandler_ { } else { // for another bot account // we only want the main bot to be able to invite virtual people - if (event.sender !== `@_discord_bot:${process.env.MATRIX_HOMESERVER}`) - return; + if (event.sender !== `@_discord_bot:${MATRIX_HOMESERVER}`) return; const user = await Matrix.for( event .state_key!.replace("@", "") - .replace(":" + process.env.MATRIX_HOMESERVER, "") as any + .replace(":" + MATRIX_HOMESERVER, "") as any ); await user.joinRoom(event.room_id); } @@ -140,41 +143,48 @@ class MatrixHandler_ { } async handleRoomJoin(event: MatrixRoomMembership) { - const linkedDiscord = await this.getDiscordFromAlias(event.room_id); - if (!linkedDiscord) return; - - const profile = await Matrix.getUserProfile(event.sender); - - await Discord.sendMessage( - linkedDiscord.guildId, - linkedDiscord.channelId, - profile, - ":arrow_right: *joined room*" - ); + // const linkedDiscord = await this.getDiscordFromAlias(event.room_id); + // if (!linkedDiscord) return; + // const profile = await Matrix.getUserProfile(event.sender); + // await Discord.sendMessage( + // linkedDiscord.guildId, + // linkedDiscord.channelId, + // profile, + // ":arrow_right: *joined room*" + // ); } async handleRoomLeave(event: MatrixRoomMembership) { - const linkedDiscord = await this.getDiscordFromAlias(event.room_id); - if (!linkedDiscord) return; - - const profile = await Matrix.getUserProfile(event.sender); - - await Discord.sendMessage( - linkedDiscord.guildId, - linkedDiscord.channelId, - profile, - ":arrow_left: *left room*" - ); + // const linkedDiscord = await this.getDiscordFromAlias(event.room_id); + // if (!linkedDiscord) return; + // const profile = await Matrix.getUserProfile(event.sender); + // await Discord.sendMessage( + // linkedDiscord.guildId, + // linkedDiscord.channelId, + // profile, + // ":arrow_left: *left room*" + // ); } async handleNewMessage(message: MatrixRoomMessage) { - const linkedDiscord = await this.getDiscordFromAlias(message.room_id); - if (!linkedDiscord) return; + const bridged = await Glue.getBridges({ + matrix_room: message.room_id, + }); + if (!bridged.length) return; const inReplyTo_eventID = "m.relates_to" in message.content && "m.in_reply_to" in message.content["m.relates_to"] && message.content["m.relates_to"]["m.in_reply_to"]?.event_id; + + /** + * If incoming message is in reply to another message, we need to find the possible bridged message + * + * Steps: + * 1. Check that incoming message is "in reply to" + * 2. Look for (matrix -> discord) message -> return discord id + * 3. Look for (discord -> matrix) message -> return discord id + */ const inReplyTo = inReplyTo_eventID && (( @@ -192,50 +202,57 @@ class MatrixHandler_ { switch (message.content.msgtype) { case "m.text": { - const discMsg = await Discord.sendMessage( - linkedDiscord.guildId, - linkedDiscord.channelId, - profile, - Discord.matrixToDiscord(message), - { - replyTo: inReplyTo || undefined, + for (const room of bridged) { + try { + const discMsg = await Discord.sendMessage( + room.discord_guild_id, + room.discord_channel_id, + profile, + Discord.matrixToDiscord(message), + { + replyTo: inReplyTo || undefined, + } + ); + + await prisma.matrixMessages.create({ + data: { + matrix_event_id: message.event_id, + matrix_room_id: message.room_id, + discord_id: discMsg.id, + discord_channel_id: discMsg.channelId, + discord_guild_id: discMsg.guildId!, + }, + }); + } catch (e) { + console.error("Failed to bridge to room", room, e); } - ); - - await prisma.matrixMessages.create({ - data: { - matrix_event_id: message.event_id, - discord_id: discMsg.id, - discord_channel_id: discMsg.channelId, - discord_guild_id: discMsg.guildId!, - }, - }); - break; - } - case "m.image": { - const discMsg = await Discord.sendMessage( - linkedDiscord.guildId, - linkedDiscord.channelId, - profile, - Discord.matrixToDiscord(message), - { - replyTo: inReplyTo || undefined, - attachments: [ - { url: Matrix.getMediaURL(message.content.url as any) as string }, - ], - } - ); - - await prisma.matrixMessages.create({ - data: { - matrix_event_id: message.event_id, - discord_id: discMsg.id, - discord_channel_id: discMsg.channelId, - discord_guild_id: discMsg.guildId!, - }, - }); + } break; } + // case "m.image": { + // const discMsg = await Discord.sendMessage( + // linkedDiscord.guildId, + // linkedDiscord.channelId, + // profile, + // Discord.matrixToDiscord(message), + // { + // replyTo: inReplyTo || undefined, + // attachments: [ + // { url: Matrix.getMediaURL(message.content.url as any) as string }, + // ], + // } + // ); + + // await prisma.matrixMessages.create({ + // data: { + // matrix_event_id: message.event_id, + // discord_id: discMsg.id, + // discord_channel_id: discMsg.channelId, + // discord_guild_id: discMsg.guildId!, + // }, + // }); + // break; + // } default: console.warn("Unhandled new room message type", message); } @@ -251,20 +268,24 @@ class MatrixHandler_ { switch (message.content.msgtype) { case "m.text": { - const db_message = await prisma.matrixMessages.findFirst({ + const db_messages = await prisma.matrixMessages.findMany({ where: { matrix_event_id: message.content["m.relates_to"]!.event_id, }, }); - if (!db_message) return; - const discMsg = await Discord.editMessage( - db_message.discord_guild_id, - db_message.discord_channel_id, - db_message.discord_id, - message.content["m.new_content"]!.body - ); - console.log(discMsg); + for (const db_message of db_messages) { + try { + const discEdit = await Discord.editMessage( + db_message.discord_guild_id, + db_message.discord_channel_id, + db_message.discord_id, + message.content["m.new_content"]!.body + ); + } catch (e) { + console.error("Edit message discord error", db_message, e); + } + } break; } default: @@ -273,60 +294,52 @@ class MatrixHandler_ { } async handleDeleteEvent(event: MatrixRoomRedaction) { - const db_message = await prisma.matrixMessages.findFirst({ + const db_matrix = await prisma.matrixMessages.findMany({ where: { matrix_event_id: event.content.redacts, }, }); - if (db_message) { - // this is a matrix -> discord message - await Discord.deleteMessage( - db_message.discord_guild_id, - db_message.discord_channel_id, - db_message.discord_id - ).catch((e) => { - console.error("Failed to delete message", db_message.discord_id, e); - }); - } else { - // this might be a discord -> matrix message - - const db2_message = await prisma.discordMessages.findFirst({ - where: { - matrix_event_id: event.content.redacts, - }, - }); - - if (db2_message) { - // this *is* a discord -> matrix message + // clean matrix -> discord message + for (const db_message of db_matrix) { + try { + // delete message on the discord side + await Discord.deleteMessage( + db_message.discord_guild_id, + db_message.discord_channel_id, + db_message.discord_id + ); + } catch (e) { + console.error("Failed to delete message", db_message, e); + } + } + + const db_discord = await prisma.discordMessages.findMany({ + where: { + matrix_event_id: event.content.redacts, + }, + }); + // clean discord -> matrix message + for (const db_message of db_discord) { + try { await prisma.discordMessages.delete({ - where: { discord_id: db2_message.discord_id }, + where: { + id: db_message.id, + }, }); - + // delete message on the discord side await Discord.deleteMessage( - db2_message.discord_guild_id, - db2_message.discord_channel_id, - db2_message.discord_id, + db_message.discord_guild_id, + db_message.discord_channel_id, + db_message.discord_id, false - ).catch((e) => { - console.error("Failed to delete message", db2_message.discord_id, e); - }); + ); + } catch (e) { + console.error("Failed to delete message", db_message, e); } } } - - async getDiscordFromAlias(room_id: string) { - const aliases = await Matrix.getAliasesForRoom(room_id); - if (!aliases[0]) return undefined; - - const [guildId, channelId] = aliases[0] - .replace("#_discord_", "") - .replace(/:(.*)/g, "") - .split("_"); - - return { guildId, channelId }; - } } export const MatrixHandler = new MatrixHandler_(); diff --git a/src/types/env.ts b/src/types/env.ts index 6c0687d..a00f739 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -1,7 +1,30 @@ declare global { namespace NodeJS { interface ProcessEnv { + NODE_ENV: "development" | "production"; PORT: string; + + /** + * Publicly accessable hostname + */ + HOST: string; + + /** + * [development] SSL port + * Matrix federation requires SSL + */ + DEV_SSL_PORT?: string; + /** + * [development] SSL key path + * Path to self-signed key + */ + DEV_SSL_KEY_PATH?: string; + /** + * [development] SSL certificate path + * Path to self-signed cert + */ + DEV_SSL_CERT_PATH?: string; + /** * Outgoing token (-> matrix) */ @@ -14,7 +37,15 @@ declare global { * Incoming token (-> express) */ MATRIX_HS_TOKEN: string; + /** + * Way to connect to homeserver + */ MATRIX_HOMESERVER: string; + /** + * Hostname that should be used instead of MATRIX_HOMESERVER + * @example `localhost` when MATRIX_HOMESERVER is `localhost:8008` + */ + MATRIX_HOMESERVER_HOST?: string; DISCORD_TOKEN: string; /** diff --git a/src/webserver.ts b/src/webserver.ts index c8799d2..4689879 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -5,10 +5,14 @@ import express, { NextFunction, Router, } from "express"; +import https from "node:https"; +import { Readable } from "node:stream"; +import fs from "node:fs"; import { MatrixError } from "./types/matrix"; import { MatrixHandler } from "./matrix"; import bodyParser from "body-parser"; import morgan from "morgan"; +import { Discord } from "./discord"; class Webserver_ { app: Express; @@ -19,7 +23,7 @@ class Webserver_ { this.app.use(morgan("tiny")); this.app.get("/", (req, res) => { - res.send("matrix-discord-bridge"); + res.send("matrix-discord-bridge " + (req.secure ? "using ssl" : "")); }); this.setupMatrixRoutes(); @@ -27,6 +31,41 @@ class Webserver_ { this.app.listen(process.env.PORT, () => { console.log("Listening on :" + process.env.PORT); }); + + if ( + process.env.NODE_ENV === "development" && + !isNaN(parseInt(process.env.DEV_SSL_PORT!)) && + process.env.DEV_SSL_KEY_PATH && + process.env.DEV_SSL_CERT_PATH + ) { + if ( + !fs.existsSync(process.env.DEV_SSL_KEY_PATH) || + !fs.existsSync(process.env.DEV_SSL_CERT_PATH) + ) { + console.warn( + "[DEV] SSL proxy could not start; couldn't find DEV_SSL_KEY_PATH or DEV_SSL_CERT_PATH" + ); + } else { + https + .createServer( + { + key: fs.readFileSync(process.env.DEV_SSL_KEY_PATH), + cert: fs.readFileSync(process.env.DEV_SSL_CERT_PATH), + }, + this.app + ) + .listen( + parseInt(process.env.DEV_SSL_PORT!), + "0.0.0.0", + undefined, + () => { + console.log( + "[DEV] SSL proxy running on :" + process.env.DEV_SSL_PORT + ); + } + ); + } + } } checkAuth(req: Request, res: Response, next: NextFunction) { @@ -51,23 +90,41 @@ class Webserver_ { }); router.put("/transactions/:txnId", async (req, res) => { - const { events } = req.body; + try { + const { events } = req.body; - await MatrixHandler.handleTransaction(req.params.txnId, events); + await MatrixHandler.handleTransaction(req.params.txnId, events); - res.status(200).json({}); + res.status(200).json({}); + } catch (e) { + // TODO: Error log + console.error(req.url, e); + res.status(500).json({}); + } }); router.get("/users/:userId", async (req, res) => { - const data = await MatrixHandler.handleUserQuery(req.params.userId); - - res.status(data.status).json("data" in data ? data.data : {}); + try { + const data = await MatrixHandler.handleUserQuery(req.params.userId); + + res.status(data.status).json("data" in data ? data.data : {}); + } catch (e) { + // TODO: Error log + console.error(req.url, e); + res.status(500).json({}); + } }); router.get("/rooms/:alias", async (req, res) => { - const data = await MatrixHandler.handleRoomQuery(req.params.alias); - - res.status(data.status).json("data" in data ? data.data : {}); + try { + const data = await MatrixHandler.handleRoomQuery(req.params.alias); + + res.status(data.status).json("data" in data ? data.data : {}); + } catch (e) { + // TODO: Error log + console.error(req.url, e); + res.status(500).json({}); + } }); // #region /thirdpart/{location,protocol} -- unused @@ -107,6 +164,49 @@ class Webserver_ { }); this.app.use("/_matrix/app/v1", router); + + this.app.get( + "/_matrix/media/v3/download/:hostname/:name", + async (req, res) => { + if (req.params.hostname !== process.env.HOST) { + return res.status(400).json({ + errcode: "M_ERROR", + error: "This service does not proxy external MXCs", + }); + } + + const [guild_id, channel_id, message_id, attachment_id] = Buffer.from( + req.params.name, + "base64url" + ) + .toString() + .split(","); + + const attachment = await Discord.getAttachmentURL( + guild_id, + channel_id, + message_id, + attachment_id + ); + if (!attachment) { + return res.status(404).json({}); + } + + if (req.query.allow_redirect) { + // if this is set, the server/client accepts redirects + return res.redirect(307, attachment); + } + + // redirects not supported, so we have to proxy it ourselves + fetch(attachment).then((proxy) => { + if (proxy.status === 200 && proxy.body) { + Readable.fromWeb(proxy.body as any).pipe(res); + } else { + res.send("bad request"); + } + }); + } + ); } } -- GitLab From aa59e1e57fb3739e761f27e959395c390d4c5422 Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 14 Nov 2024 21:27:13 -0700 Subject: [PATCH 02/21] add matrix -> discord image --- src/lib/matrix.ts | 2 +- src/matrix.ts | 59 ++++++++++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index 081d98d..59ebafa 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -403,7 +403,7 @@ export class Matrix { return mxc.replace( "mxc://", - `https://${process.env.MATRIX_HOMESERVER}/_matrix/media/v3/download/` + `https://${MATRIX_HOMESERVER}/_matrix/media/v3/download/` ); } diff --git a/src/matrix.ts b/src/matrix.ts index 4ab5bd0..a7cd41c 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -229,30 +229,41 @@ class MatrixHandler_ { } break; } - // case "m.image": { - // const discMsg = await Discord.sendMessage( - // linkedDiscord.guildId, - // linkedDiscord.channelId, - // profile, - // Discord.matrixToDiscord(message), - // { - // replyTo: inReplyTo || undefined, - // attachments: [ - // { url: Matrix.getMediaURL(message.content.url as any) as string }, - // ], - // } - // ); - - // await prisma.matrixMessages.create({ - // data: { - // matrix_event_id: message.event_id, - // discord_id: discMsg.id, - // discord_channel_id: discMsg.channelId, - // discord_guild_id: discMsg.guildId!, - // }, - // }); - // break; - // } + case "m.image": { + for (const room of bridged) { + try { + const discMsg = await Discord.sendMessage( + room.discord_guild_id, + room.discord_channel_id, + profile, + Discord.matrixToDiscord(message), + { + replyTo: inReplyTo || undefined, + attachments: [ + { + url: Matrix.getMediaURL( + message.content.url as any + ) as string, + }, + ], + } + ); + + await prisma.matrixMessages.create({ + data: { + matrix_event_id: message.event_id, + matrix_room_id: message.room_id, + discord_id: discMsg.id, + discord_channel_id: discMsg.channelId, + discord_guild_id: discMsg.guildId!, + }, + }); + } catch (e) { + console.error("Failed to bridge to room", room, e); + } + } + break; + } default: console.warn("Unhandled new room message type", message); } -- GitLab From 4dfed9c9f7fdd5a6ec697f76607f390ade756919 Mon Sep 17 00:00:00 2001 From: Grant <3380410-grahhnt@users.noreply.gitlab.com> Date: Sun, 17 Nov 2024 00:55:28 -0700 Subject: [PATCH 03/21] add dev:ssl utility script --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 1dac9e8..d943653 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "dev": "tsx watch -r dotenv/config src/index.ts", "build": "tsc", - "start": "node dist/index.js" + "start": "node dist/index.js", + "dev:ssl": "openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout selfsigned.key -out selfsigned.crt" }, "keywords": [], "author": "", -- GitLab From a7b110b1cb4615bb2b6dc26adb0e46b8d27d4789 Mon Sep 17 00:00:00 2001 From: Grant <3380410-grahhnt@users.noreply.gitlab.com> Date: Sun, 17 Nov 2024 02:19:26 -0700 Subject: [PATCH 04/21] matrix -> discord replies --- package-lock.json | 38 ++++++++++++++-- package.json | 1 + .../migrations/20241117083622_/migration.sql | 37 ++++++++++++++++ prisma/schema.prisma | 12 ++++-- src/discord.ts | 38 ++++++++++++---- src/matrix.ts | 43 +++++++++++++------ 6 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 prisma/migrations/20241117083622_/migration.sql diff --git a/package-lock.json b/package-lock.json index 21aecbe..e43caf7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@prisma/client": "^5.15.0", "body-parser": "^1.20.2", + "discord-api-types": "^0.37.105", "discord.js": "^14.15.3", "eventemitter3": "^5.0.1", "express": "^4.19.2", @@ -52,6 +53,12 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/builders/node_modules/discord-api-types": { + "version": "0.37.83", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", + "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==", + "license": "MIT" + }, "node_modules/@discordjs/collection": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.5.3.tgz", @@ -74,6 +81,12 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/formatters/node_modules/discord-api-types": { + "version": "0.37.83", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", + "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==", + "license": "MIT" + }, "node_modules/@discordjs/rest": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-2.3.0.tgz", @@ -107,6 +120,12 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/rest/node_modules/discord-api-types": { + "version": "0.37.83", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", + "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==", + "license": "MIT" + }, "node_modules/@discordjs/util": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.1.0.tgz", @@ -151,6 +170,12 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/@discordjs/ws/node_modules/discord-api-types": { + "version": "0.37.83", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", + "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==", + "license": "MIT" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz", @@ -1471,9 +1496,10 @@ } }, "node_modules/discord-api-types": { - "version": "0.37.83", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", - "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==" + "version": "0.37.105", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.105.tgz", + "integrity": "sha512-TSNlLF5Q9vFLMeHjYskhmDj/zCQ4dFA+OLxQrHUypGW48gt8ttGaB+opCD9w3Zkq1otyoBoetrwaoDFZAFuGng==", + "license": "MIT" }, "node_modules/discord.js": { "version": "14.15.3", @@ -1500,6 +1526,12 @@ "url": "https://github.com/discordjs/discord.js?sponsor" } }, + "node_modules/discord.js/node_modules/discord-api-types": { + "version": "0.37.83", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.83.tgz", + "integrity": "sha512-urGGYeWtWNYMKnYlZnOnDHm8fVRffQs3U0SpE8RHeiuLKb/u92APS8HoQnPTFbnXmY1vVnXjXO4dOxcAn3J+DA==", + "license": "MIT" + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", diff --git a/package.json b/package.json index d943653..8aaa879 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "dependencies": { "@prisma/client": "^5.15.0", "body-parser": "^1.20.2", + "discord-api-types": "^0.37.105", "discord.js": "^14.15.3", "eventemitter3": "^5.0.1", "express": "^4.19.2", diff --git a/prisma/migrations/20241117083622_/migration.sql b/prisma/migrations/20241117083622_/migration.sql new file mode 100644 index 0000000..5950d98 --- /dev/null +++ b/prisma/migrations/20241117083622_/migration.sql @@ -0,0 +1,37 @@ +/* + Warnings: + + - Added the required column `discord_author_id` to the `DiscordMessages` table without a default value. This is not possible if the table is not empty. + - Added the required column `matrix_author_id` to the `MatrixMessages` table without a default value. This is not possible if the table is not empty. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_DiscordMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL, + "discord_author_id" TEXT NOT NULL, + "attachment_id" TEXT, + "matrix_event_id" TEXT NOT NULL, + "matrix_room_id" TEXT NOT NULL +); +INSERT INTO "new_DiscordMessages" ("attachment_id", "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id", "matrix_room_id") SELECT "attachment_id", "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id", "matrix_room_id" FROM "DiscordMessages"; +DROP TABLE "DiscordMessages"; +ALTER TABLE "new_DiscordMessages" RENAME TO "DiscordMessages"; +CREATE TABLE "new_MatrixMessages" ( + "id" TEXT NOT NULL PRIMARY KEY, + "matrix_event_id" TEXT NOT NULL, + "matrix_room_id" TEXT NOT NULL, + "matrix_author_id" TEXT NOT NULL, + "discord_id" TEXT NOT NULL, + "discord_channel_id" TEXT NOT NULL, + "discord_guild_id" TEXT NOT NULL +); +INSERT INTO "new_MatrixMessages" ("discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id", "matrix_room_id") SELECT "discord_channel_id", "discord_guild_id", "discord_id", "id", "matrix_event_id", "matrix_room_id" FROM "MatrixMessages"; +DROP TABLE "MatrixMessages"; +ALTER TABLE "new_MatrixMessages" RENAME TO "MatrixMessages"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index fda87d7..37c7c0f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -26,10 +26,12 @@ model DiscordWebhooks { // discord -> matrix model DiscordMessages { - id String @id @default(uuid()) // one message may get represented as multiple + id String @id @default(uuid()) // one message may get represented as multiple + discord_id String discord_channel_id String discord_guild_id String + discord_author_id String attachment_id String? matrix_event_id String @@ -38,9 +40,11 @@ model DiscordMessages { // matrix -> discord model MatrixMessages { - id String @id @default(uuid()) // one message may get represented as multiple - matrix_event_id String - matrix_room_id String + id String @id @default(uuid()) // one message may get represented as multiple + + matrix_event_id String + matrix_room_id String + matrix_author_id String discord_id String discord_channel_id String diff --git a/src/discord.ts b/src/discord.ts index e379c56..ede0f09 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -124,9 +124,10 @@ class Discord_ { if (result.status === 200) { return prisma.discordMessages.create({ data: { + discord_id: message.id, discord_guild_id: message.guildId!, discord_channel_id: message.channelId, - discord_id: message.id, + discord_author_id: message.author.id, matrix_event_id: result.data.event_id, matrix_room_id: room.matrix_room_id, }, @@ -162,9 +163,10 @@ class Discord_ { if (result.status === 200) { return prisma.discordMessages.create({ data: { + discord_id: message.id, discord_guild_id: message.guildId!, discord_channel_id: message.channelId, - discord_id: message.id, + discord_author_id: message.author.id, attachment_id: id, matrix_event_id: result.data.event_id, matrix_room_id: room.matrix_room_id, @@ -362,10 +364,16 @@ class Discord_ { user: IMatrixUser, content: string, options: { - /** - * Discord Message ID - */ - replyTo?: string; + replyTo?: { + /** + * Discord Message ID + */ + messageId: string; + /** + * Display name for a user replying to + */ + who: string; + }; attachments?: { url: string }[]; } = {} ) { @@ -384,14 +392,28 @@ class Discord_ { let components = []; if (options.replyTo) { + /** + * Is the replyTo a Discord message? + * If the replyTo user starts with an @, it's mentioning a Matrix ID + */ + const isDiscord = !options.replyTo.who.startsWith("@"); + const replyToWho = isDiscord + ? "@" + (await this.client.users.fetch(options.replyTo.who)).username + : options.replyTo.who; + components.push( new ActionRowBuilder().addComponents( new ButtonBuilder() .setStyle(ButtonStyle.Link) .setURL( - `https://discord.com/channels/${guild_id}/${channel_id}/${options.replyTo}` + `https://discord.com/channels/${guild_id}/${channel_id}/${options.replyTo.messageId}` ) - .setLabel("In Reply To") + .setLabel("In Reply To"), + new ButtonBuilder() + .setStyle(ButtonStyle.Secondary) + .setCustomId("nonce") + .setDisabled(true) + .setLabel(replyToWho) ) ); } diff --git a/src/matrix.ts b/src/matrix.ts index a7cd41c..da2c108 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -185,18 +185,33 @@ class MatrixHandler_ { * 2. Look for (matrix -> discord) message -> return discord id * 3. Look for (discord -> matrix) message -> return discord id */ - const inReplyTo = - inReplyTo_eventID && - (( - await prisma.matrixMessages.findFirst({ + let inReplyTo: NonNullable< + Parameters<(typeof Discord)["sendMessage"]>[4] + >["replyTo"] = undefined; + + if (inReplyTo_eventID) { + const matrixMessage = await prisma.matrixMessages.findFirst({ + where: { matrix_event_id: inReplyTo_eventID }, + }); + + if (matrixMessage) { + inReplyTo = { + messageId: matrixMessage.discord_id, + who: matrixMessage.matrix_author_id, + }; + } else { + const discordMessage = await prisma.discordMessages.findFirst({ where: { matrix_event_id: inReplyTo_eventID }, - }) - )?.discord_id || - ( - await prisma.discordMessages.findFirst({ - where: { matrix_event_id: inReplyTo_eventID }, - }) - )?.discord_id); + }); + + if (discordMessage) { + inReplyTo = { + messageId: discordMessage.discord_id, + who: discordMessage.discord_author_id, + }; + } + } + } const profile = await Matrix.getUserProfile(message.sender); @@ -210,7 +225,7 @@ class MatrixHandler_ { profile, Discord.matrixToDiscord(message), { - replyTo: inReplyTo || undefined, + replyTo: inReplyTo, } ); @@ -218,6 +233,7 @@ class MatrixHandler_ { data: { matrix_event_id: message.event_id, matrix_room_id: message.room_id, + matrix_author_id: message.sender, discord_id: discMsg.id, discord_channel_id: discMsg.channelId, discord_guild_id: discMsg.guildId!, @@ -238,7 +254,7 @@ class MatrixHandler_ { profile, Discord.matrixToDiscord(message), { - replyTo: inReplyTo || undefined, + replyTo: inReplyTo, attachments: [ { url: Matrix.getMediaURL( @@ -253,6 +269,7 @@ class MatrixHandler_ { data: { matrix_event_id: message.event_id, matrix_room_id: message.room_id, + matrix_author_id: message.sender, discord_id: discMsg.id, discord_channel_id: discMsg.channelId, discord_guild_id: discMsg.guildId!, -- GitLab From a044e544d0f0e231dc11181121921def64b66685 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 12:29:31 -0700 Subject: [PATCH 05/21] discord message replies (discord & matrix) --- src/discord.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/discord.ts b/src/discord.ts index ede0f09..227c6c2 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -105,6 +105,22 @@ class Discord_ { matrixClient.username ); + const reference = await message.fetchReference().catch((e) => null); + const inReplyTo_eventID = + reference && + ((await prisma.discordMessages.findFirst({ + where: { + discord_id: reference.id, + discord_channel_id: reference.channelId, + }, + })) || + (await prisma.matrixMessages.findFirst({ + where: { + discord_id: reference.id, + discord_channel_id: reference.channelId, + }, + }))); + await matrixClient .sendEvent( room.matrix_room_id, @@ -116,6 +132,13 @@ class Discord_ { "m.mentions": { user_ids: Matrix.scanForMatrixMentions(message.content), }, + "m.relates_to": inReplyTo_eventID + ? { + "m.in_reply_to": { + event_id: inReplyTo_eventID.matrix_event_id, + }, + } + : undefined, }, }, [room.matrix_room_id, message.id].join(",") -- GitLab From 1f07b98fc686348d57908882a030d2c8594a6cd5 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 13:10:13 -0700 Subject: [PATCH 06/21] discord add video, audio, file --- src/discord.ts | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index 227c6c2..b1f57e8 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -161,24 +161,39 @@ class Discord_ { }); for (const [id, attachment] of message.attachments) { + let attachmentType: MatrixRoomMessage["content"]["msgtype"] = + "m.file"; + + if (attachment.contentType?.startsWith("image/")) { + attachmentType = "m.image"; + } + if (attachment.contentType?.startsWith("video/")) { + attachmentType = "m.video"; + } + if (attachment.contentType?.startsWith("audio/")) { + attachmentType = "m.audio"; + } + + let content: MatrixRoomMessage["content"] = { + msgtype: attachmentType, + body: "attachment." + attachment.name.split(".").at(-1), + url: `mxc://${process.env.HOST}/${Buffer.from([message.guildId, message.channelId, message.id, id].join(",")).toString("base64url")}`, + info: { + mimetype: attachment.contentType!, + h: attachment.height || undefined, + w: attachment.width || undefined, + }, + "m.mentions": { + user_ids: [], + }, + } as any; // no idea why the type isn't valid + await matrixClient .sendEvent( room.matrix_room_id, { type: "m.room.message", - content: { - msgtype: "m.image", - body: "attachment.png", - url: `mxc://${process.env.HOST}/${Buffer.from([message.guildId, message.channelId, message.id, id].join(",")).toString("base64url")}`, - info: { - mimetype: attachment.contentType!, - // w: 128, - // h: 128, - }, - "m.mentions": { - user_ids: [], - }, - }, + content, }, [room.matrix_room_id, message.id, id].join(",") ) -- GitLab From db97b0795b49926289d686e9a20c2ab2e4c35bd6 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 13:12:12 -0700 Subject: [PATCH 07/21] customid --- src/discord.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/discord.ts b/src/discord.ts index b1f57e8..99fa5c2 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -449,7 +449,7 @@ class Discord_ { .setLabel("In Reply To"), new ButtonBuilder() .setStyle(ButtonStyle.Secondary) - .setCustomId("nonce") + .setCustomId("dummy") .setDisabled(true) .setLabel(replyToWho) ) -- GitLab From bb909141dafcb33c38ba764a88bd0fcc4690e1c4 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 14:33:59 -0700 Subject: [PATCH 08/21] add matrix -> discord video, image, file --- src/discord.ts | 55 +++++++++++++++++++++++++++++++++++------------ src/lib/matrix.ts | 46 +++++++++++++++++++++++++++------------ src/matrix.ts | 17 +++++++++------ 3 files changed, 83 insertions(+), 35 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index 99fa5c2..46858e0 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -412,7 +412,10 @@ class Discord_ { */ who: string; }; - attachments?: { url: string }[]; + /** + * IMPORTANT: attachment URLs are fetched and streamed to discord + */ + attachment?: { url: string; name: string }; } = {} ) { const webhook = await this.getWebhookFor(guild_id, channel_id); @@ -456,19 +459,43 @@ class Discord_ { ); } - let embeds = []; + let messageContent = { + username: displayName, + avatarURL: Matrix.getMediaURL(user.avatar_url), + content, + components, + allowedMentions: { + parse: ["users"], + roles: [], + } as const, + }; - if (options.attachments && options.attachments?.length > 0) { - embeds.push({ - image: { - url: options.attachments[0].url, - }, - footer: - options.attachments.length - 1 > 0 - ? { - text: "+" + (options.attachments.length - 1) + " others", - } - : undefined, + let messageId: string | undefined; + + let files: { attachment: Buffer; name: string }[] = []; + + if (options.attachment) { + messageId = ( + await webhook.send({ + ...messageContent, + content: "*Uploading attachment*", + }) + ).id; + + const stream = await fetch(options.attachment.url).then((req) => + req.arrayBuffer() + ); + + files.push({ + attachment: Buffer.from(stream), + name: options.attachment.name, + }); + } + + if (messageId) { + return webhook.editMessage(messageId, { + ...messageContent, + files, }); } @@ -477,7 +504,7 @@ class Discord_ { avatarURL: Matrix.getMediaURL(user.avatar_url), content, components, - embeds, + files, allowedMentions: { parse: ["users"], roles: [], diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index 59ebafa..1269abc 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -50,6 +50,28 @@ export class Matrix { ...args ); + /** + * Get homeserver access + * + * * internal: used by this bridge internally + * * external: passed to external service (like discord or client) + * + * @param use + * @returns + */ + private static getHomeserver(use: "internal" | "external") { + switch (use) { + case "internal": { + let protocol = + process.env.NODE_ENV === "development" ? "http" : "https"; + return protocol + "://" + process.env.MATRIX_HOMESERVER; + } + case "external": { + return "https://" + MATRIX_HOMESERVER; + } + } + } + private static async fetch( endpoint: `/_matrix${string}`, method = "GET", @@ -60,18 +82,14 @@ export class Matrix { > { console.log("[->Matrix] " + method + " " + endpoint); - let protocol = process.env.NODE_ENV === "development" ? "http" : "https"; - const req = await fetch( - `${protocol}://${process.env.MATRIX_HOMESERVER}${endpoint}`, - { - method, - headers: { - "Content-Type": "application/json", - Authorization: "Bearer " + MATRIX_AS_TOKEN, - }, - body: JSON.stringify(data), - } - ); + const req = await fetch(`${this.getHomeserver("internal")}${endpoint}`, { + method, + headers: { + "Content-Type": "application/json", + Authorization: "Bearer " + MATRIX_AS_TOKEN, + }, + body: JSON.stringify(data), + }); const res = await req.json(); @@ -398,12 +416,12 @@ export class Matrix { * @param mxc * @returns */ - static getMediaURL(mxc?: MXC) { + static getMediaURL(mxc?: MXC, use: "internal" | "external" = "external") { if (!mxc) return undefined; return mxc.replace( "mxc://", - `https://${MATRIX_HOMESERVER}/_matrix/media/v3/download/` + `${this.getHomeserver(use)}/_matrix/media/v3/download/` ); } diff --git a/src/matrix.ts b/src/matrix.ts index da2c108..62774f0 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -245,6 +245,9 @@ class MatrixHandler_ { } break; } + case "m.file": + case "m.video": + case "m.audio": case "m.image": { for (const room of bridged) { try { @@ -255,13 +258,13 @@ class MatrixHandler_ { Discord.matrixToDiscord(message), { replyTo: inReplyTo, - attachments: [ - { - url: Matrix.getMediaURL( - message.content.url as any - ) as string, - }, - ], + attachment: { + url: Matrix.getMediaURL( + message.content.url as any, + "internal" + ) as string, + name: message.content.body, + }, } ); -- GitLab From 3d8ee232d8f6f2121f53a6eceed8243688835ffd Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 14:51:16 -0700 Subject: [PATCH 09/21] discord -> matrix profile syncing --- src/discord.ts | 7 ++++-- src/webserver.ts | 62 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index 46858e0..aa0a00a 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -92,7 +92,10 @@ class Discord_ { let matrixClient; try { matrixClient = await Matrix.for(`_discord_${message.author.id}`); - await matrixClient.syncProfile(message.author.displayName); + await matrixClient.syncProfile( + message.author.displayName, + `mxc://${process.env.HOST}/${Buffer.from(["avatar", message.author.id].join(",")).toString("base64url")}` + ); } catch (e) { console.error("Failed to get bridged ghost user", e); return; @@ -177,7 +180,7 @@ class Discord_ { let content: MatrixRoomMessage["content"] = { msgtype: attachmentType, body: "attachment." + attachment.name.split(".").at(-1), - url: `mxc://${process.env.HOST}/${Buffer.from([message.guildId, message.channelId, message.id, id].join(",")).toString("base64url")}`, + url: `mxc://${process.env.HOST}/${Buffer.from(["attachment", message.guildId, message.channelId, message.id, id].join(",")).toString("base64url")}`, info: { mimetype: attachment.contentType!, h: attachment.height || undefined, diff --git a/src/webserver.ts b/src/webserver.ts index 4689879..3f7efaf 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -175,30 +175,64 @@ class Webserver_ { }); } - const [guild_id, channel_id, message_id, attachment_id] = Buffer.from( - req.params.name, - "base64url" - ) + const parts = Buffer.from(req.params.name, "base64url") .toString() .split(","); - const attachment = await Discord.getAttachmentURL( - guild_id, - channel_id, - message_id, - attachment_id - ); - if (!attachment) { - return res.status(404).json({}); + const mode = parts.shift(); + let targetURL: string; + + switch (mode) { + case "attachment": { + const [guild_id, channel_id, message_id, attachment_id] = parts; + + const attachment = await Discord.getAttachmentURL( + guild_id, + channel_id, + message_id, + attachment_id + ); + if (!attachment) { + return res.status(404).json({}); + } + + targetURL = attachment; + break; + } + case "avatar": { + const [userid] = parts; + + const user = await Discord.client.users + .fetch(userid) + .catch((e) => null); + if (!user) { + return res.status(404).json({}); + } + + const avatarURL = user.avatarURL({ + extension: "png", + forceStatic: true, + }); + if (!avatarURL) { + return res.status(404).json({}); + } + + targetURL = avatarURL; + break; + } + default: + return res + .status(404) + .json({ errcode: "M_UNKNOWN", error: "Unknown media" }); } if (req.query.allow_redirect) { // if this is set, the server/client accepts redirects - return res.redirect(307, attachment); + return res.redirect(307, targetURL); } // redirects not supported, so we have to proxy it ourselves - fetch(attachment).then((proxy) => { + fetch(targetURL).then((proxy) => { if (proxy.status === 200 && proxy.body) { Readable.fromWeb(proxy.body as any).pipe(res); } else { -- GitLab From 179fa4fb8f21c6cf6d891528be7aadbb2c279ee5 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 17 Nov 2024 16:55:43 -0700 Subject: [PATCH 10/21] add matrix well known --- src/webserver.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webserver.ts b/src/webserver.ts index 3f7efaf..74fb64f 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -165,6 +165,12 @@ class Webserver_ { this.app.use("/_matrix/app/v1", router); + this.app.get("/.well-known/matrix/server", (req, res) => { + res.json({ + "m.server": process.env.HOST, + }); + }); + this.app.get( "/_matrix/media/v3/download/:hostname/:name", async (req, res) => { -- GitLab From e8bf0a06e2af980b060d9527ee0ff268581a770a Mon Sep 17 00:00:00 2001 From: Grant Date: Sat, 23 Nov 2024 21:25:37 -0700 Subject: [PATCH 11/21] handle attachment errors to discord --- src/discord.ts | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index aa0a00a..aeb3adc 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -1,4 +1,5 @@ import { + APIEmbed, ActionRowBuilder, ButtonBuilder, ButtonStyle, @@ -6,6 +7,7 @@ import { ChannelType, Client, ComponentType, + EmbedBuilder, GatewayIntentBits, GuildBasedChannel, GuildChannelResolvable, @@ -71,7 +73,17 @@ class Discord_ { const guild = await this.client.guilds.fetch(guild_id); const channel = await guild.channels.fetch(channel_id); if (!channel || channel.type !== ChannelType.GuildText) return; - const message = await channel.messages.fetch(message_id); + // force the messages to be fetched from scratch to avoid caching invalid attachment URLs + const messages = await channel.messages.fetch({ + around: message_id, + cache: false, + }); + + const message = messages.first(); + if (!message) { + return undefined; + } + return message.attachments.get(attachment_id)?.url; } @@ -471,6 +483,7 @@ class Discord_ { parse: ["users"], roles: [], } as const, + embeds: [] as APIEmbed[], }; let messageId: string | undefined; @@ -485,14 +498,32 @@ class Discord_ { }) ).id; - const stream = await fetch(options.attachment.url).then((req) => - req.arrayBuffer() - ); + try { + const request = await fetch(options.attachment.url); + if (request.status > 399) + throw new Error("Unknown status: " + request.status); - files.push({ - attachment: Buffer.from(stream), - name: options.attachment.name, - }); + const stream = await request.arrayBuffer(); + + files.push({ + attachment: Buffer.from(stream), + name: options.attachment.name, + }); + } catch (e) { + console.error( + "Failed to upload attachment", + guild_id, + channel_id, + user, + e + ); + messageContent.embeds.push( + new EmbedBuilder() + .setColor("Red") + .setDescription("Failed to upload attachment") + .toJSON() + ); + } } if (messageId) { -- GitLab From 261c0257563229e6d19cde12087b6ba9b1b49e71 Mon Sep 17 00:00:00 2001 From: Grant Date: Sat, 23 Nov 2024 21:27:17 -0700 Subject: [PATCH 12/21] report port in well-known/matrix/server otherwise synapse will hang --- src/webserver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webserver.ts b/src/webserver.ts index 74fb64f..2948155 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -167,7 +167,7 @@ class Webserver_ { this.app.get("/.well-known/matrix/server", (req, res) => { res.json({ - "m.server": process.env.HOST, + "m.server": process.env.HOST + ":443", }); }); -- GitLab From c22a535050d343899b20249ee1cc95d719429134 Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 24 Nov 2024 01:06:58 -0700 Subject: [PATCH 13/21] handle matrix -> discord typing --- src/lib/matrix.ts | 17 ++++++++--------- src/matrix.ts | 40 +++++++++++++++++++++++++++++++++++++++- src/types/matrix.ts | 19 +++++++++++++++++++ src/webserver.ts | 8 ++++++-- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index 1269abc..b60890c 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -40,15 +40,14 @@ export class Matrix { this.refresh_token = refresh_token; } - private fetch: (typeof Matrix)["fetch"] = (endpoint, ...args) => - Matrix.fetch( - (endpoint + - "?user_id=@" + - this.username + - ":" + - MATRIX_HOMESERVER) as any, - ...args - ); + private fetch: (typeof Matrix)["fetch"] = (_endpoint, ...args) => { + const [endpoint, rawParams] = _endpoint.split("?"); + + const params = new URLSearchParams(rawParams); + params.set("user_id", "@" + this.username + ":" + MATRIX_HOMESERVER); + + return Matrix.fetch((endpoint + "?" + params) as any, ...args); + }; /** * Get homeserver access diff --git a/src/matrix.ts b/src/matrix.ts index 62774f0..2654e67 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -4,6 +4,8 @@ import { Matrix } from "./lib/matrix"; import { prisma } from "./lib/prisma"; import { IClientEvent, + MatrixEphemeralEvent, + MatrixEphemeralTyping, MatrixError, MatrixRoomMembership, MatrixRoomMessage, @@ -19,9 +21,24 @@ const MATRIX_HOMESERVER = class MatrixHandler_ { private handledTransactionIDs: string[] = []; - async handleTransaction(txnId: string, events: IClientEvent[]) { + async handleTransaction( + txnId: string, + events: IClientEvent[], + ephemeral: MatrixEphemeralEvent[] + ) { if (this.handledTransactionIDs.indexOf(txnId) > -1) return; + for (const event of ephemeral) { + console.log("[e:Matrix->]", event); + + switch (event.type) { + case "m.typing": { + await this.handleTyping(event); + break; + } + } + } + for (const event of events) { console.log("[Matrix->]", event); @@ -94,6 +111,27 @@ class MatrixHandler_ { return { status: 200 }; } + /** + * Handle m.typing messages + * @param event + * @returns + */ + async handleTyping(event: MatrixEphemeralTyping) { + if (event.content.user_ids.length === 0) return; // discord doesn't have a "stop typing event" + + const bridges = await Glue.getBridges({ matrix_room: event.room_id }); + if (!bridges) return; + + for (const bridge of bridges) { + const { guild, channel } = await Discord.getGuildChannel( + bridge.discord_guild_id, + bridge.discord_channel_id + ); + + channel.sendTyping(); + } + } + async handleRoomMembership(event: MatrixRoomMembership) { if (event.content.membership === "invite") { // if someone is being invited, we only want to react when it's related to one of the virual accounts diff --git a/src/types/matrix.ts b/src/types/matrix.ts index 2aab4b2..0265d2f 100644 --- a/src/types/matrix.ts +++ b/src/types/matrix.ts @@ -24,6 +24,25 @@ export interface IClientEvent { unsigned?: MatrixUnsignedData; } +export interface IEphemeralEvent< + Type extends string = string, + Content = unknown, +> { + type: Type; + content: Content; +} + +export type MatrixEphemeralEvent = MatrixEphemeralTyping; + +export type MatrixEphemeralTyping = IEphemeralEvent< + "m.typing", + { + user_ids: string[]; + } +> & { + room_id: string; +}; + export type MatrixUnsignedData = { age?: number; prev_content?: Content; diff --git a/src/webserver.ts b/src/webserver.ts index 2948155..d5780ae 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -91,9 +91,13 @@ class Webserver_ { router.put("/transactions/:txnId", async (req, res) => { try { - const { events } = req.body; + const { events, "de.sorunome.msc2409.ephemeral": ephemeral } = req.body; - await MatrixHandler.handleTransaction(req.params.txnId, events); + await MatrixHandler.handleTransaction( + req.params.txnId, + events, + ephemeral + ); res.status(200).json({}); } catch (e) { -- GitLab From 838209b7a47a7a9988da637ee0594d8fd387d9fe Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 24 Nov 2024 01:31:27 -0700 Subject: [PATCH 14/21] filtering typing to ignore discord bots --- src/matrix.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/matrix.ts b/src/matrix.ts index 2654e67..61501a9 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -117,7 +117,10 @@ class MatrixHandler_ { * @returns */ async handleTyping(event: MatrixEphemeralTyping) { - if (event.content.user_ids.length === 0) return; // discord doesn't have a "stop typing event" + const user_ids = event.content.user_ids.filter( + (uid) => !uid.startsWith("@_discord_") + ); + if (user_ids.length === 0) return; // discord doesn't have a "stop typing event" const bridges = await Glue.getBridges({ matrix_room: event.room_id }); if (!bridges) return; -- GitLab From 031b436ff53b026587a07ea6f2def93874dabede Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 24 Nov 2024 01:32:04 -0700 Subject: [PATCH 15/21] discord -> matrix typing --- src/discord.ts | 22 ++++++++++++++++++++-- src/lib/matrix.ts | 11 +++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index aeb3adc..19e9e56 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -53,7 +53,7 @@ class Discord_ { this.client.on("messageCreate", this.handleMessageCreate.bind(this)); this.client.on("messageUpdate", this.handleMessageUpdate.bind(this)); this.client.on("messageDelete", this.handleMessageDelete.bind(this)); - // this.client.on("typingStart", this.handleTypingStart.bind(this)); + this.client.on("typingStart", this.handleTypingStart.bind(this)); this.client.on("ready", () => { console.log(`Logged in as ${this.client.user?.tag}`); @@ -409,7 +409,25 @@ class Discord_ { // } } - async handleTypingStart(typing: Typing) {} + async handleTypingStart(typing: Typing) { + if (typing.user.id === this.client.user!.id) return; // ignore bot's own messages + if (typing.user.bot) return; // if it's a webhook we don't care about it + if (!typing.inGuild()) return; + + const bridges = await Glue.getBridges({ + discord_guild: typing.guild.id, + discord_channel: typing.channel.id, + }); + const user = await Matrix.for(`_discord_${typing.user.id}`); + + for (const bridge of bridges) { + await Matrix.ensureUserInRoom( + bridge.matrix_room_id, + `_discord_${typing.user.id}` + ); + await user.sendTyping(bridge.matrix_room_id); + } + } async sendMessage( guild_id: string, diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index b60890c..fd0691e 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -98,6 +98,17 @@ export class Matrix { }; } + async sendTyping(roomId: string) { + return this.fetch<{}>( + `/_matrix/client/v3/rooms/${roomId}/typing/@${this.username}:${MATRIX_HOMESERVER}`, + "PUT", + { + timeout: 1000 * 10, + typing: true, + } + ); + } + /** * Send event to the server * -- GitLab From 9b85285e68b3e8ebd49afe86ea9e61cc425c6fdf Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 24 Nov 2024 19:22:43 -0700 Subject: [PATCH 16/21] [wip] matrix unknown message handling --- src/lib/matrix.ts | 22 +++++++++++++++ src/matrix.ts | 71 +++++++++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index fd0691e..729beba 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -4,6 +4,7 @@ import { IMatrixUser, MXC, MatrixRoomMembership, + MatrixRoomMessage, SentClientEvent, } from "../types/matrix"; import { FriendlyError } from "./utils"; @@ -367,6 +368,27 @@ export class Matrix { return req; } + async replyNotice(message: MatrixRoomMessage) { + return this.sendEvent( + message.room_id, + { + type: "m.room.message", + content: { + msgtype: "m.notice", + body: + "**Can't bridge** Unknown message type: " + message.content.msgtype, + "m.mentions": { user_ids: [] }, + "m.relates_to": { + "m.in_reply_to": { + event_id: message.event_id, + }, + }, + }, + }, + message.event_id + "_failure" + ); + } + /** * Get all local aliases for room_id * diff --git a/src/matrix.ts b/src/matrix.ts index 61501a9..0c83824 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -213,6 +213,14 @@ class MatrixHandler_ { }); if (!bridged.length) return; + let matrixClient; + try { + matrixClient = await Matrix.for("_discord_bot"); + } catch (e) { + console.error("Failed to get main bot account", e); + return; + } + const inReplyTo_eventID = "m.relates_to" in message.content && "m.in_reply_to" in message.content["m.relates_to"] && @@ -257,35 +265,35 @@ class MatrixHandler_ { const profile = await Matrix.getUserProfile(message.sender); switch (message.content.msgtype) { - case "m.text": { - for (const room of bridged) { - try { - const discMsg = await Discord.sendMessage( - room.discord_guild_id, - room.discord_channel_id, - profile, - Discord.matrixToDiscord(message), - { - replyTo: inReplyTo, - } - ); - - await prisma.matrixMessages.create({ - data: { - matrix_event_id: message.event_id, - matrix_room_id: message.room_id, - matrix_author_id: message.sender, - discord_id: discMsg.id, - discord_channel_id: discMsg.channelId, - discord_guild_id: discMsg.guildId!, - }, - }); - } catch (e) { - console.error("Failed to bridge to room", room, e); - } - } - break; - } + // case "m.text": { + // for (const room of bridged) { + // try { + // const discMsg = await Discord.sendMessage( + // room.discord_guild_id, + // room.discord_channel_id, + // profile, + // Discord.matrixToDiscord(message), + // { + // replyTo: inReplyTo, + // } + // ); + + // await prisma.matrixMessages.create({ + // data: { + // matrix_event_id: message.event_id, + // matrix_room_id: message.room_id, + // matrix_author_id: message.sender, + // discord_id: discMsg.id, + // discord_channel_id: discMsg.channelId, + // discord_guild_id: discMsg.guildId!, + // }, + // }); + // } catch (e) { + // console.error("Failed to bridge to room", room, e); + // } + // } + // break; + // } case "m.file": case "m.video": case "m.audio": @@ -325,8 +333,11 @@ class MatrixHandler_ { } break; } - default: + default: { console.warn("Unhandled new room message type", message); + + await matrixClient.replyNotice(message); + } } } -- GitLab From e96f5bb5b380cd9bca50cae4225901f7a7a65af4 Mon Sep 17 00:00:00 2001 From: Grant <3380410-grahhnt@users.noreply.gitlab.com> Date: Sun, 24 Nov 2024 23:57:26 -0700 Subject: [PATCH 17/21] handle unknown matrix events --- src/lib/matrix.ts | 13 +++++++++---- src/matrix.ts | 21 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/lib/matrix.ts b/src/lib/matrix.ts index 729beba..b8972fa 100644 --- a/src/lib/matrix.ts +++ b/src/lib/matrix.ts @@ -368,16 +368,21 @@ export class Matrix { return req; } - async replyNotice(message: MatrixRoomMessage) { + /** + * Reply to a message with a m.notice message + * @param message + * @param body + * @returns + */ + async replyNotice(message: MatrixRoomMessage, body: string) { return this.sendEvent( message.room_id, { type: "m.room.message", content: { msgtype: "m.notice", - body: - "**Can't bridge** Unknown message type: " + message.content.msgtype, - "m.mentions": { user_ids: [] }, + body, + "m.mentions": { user_ids: [message.sender] }, "m.relates_to": { "m.in_reply_to": { event_id: message.event_id, diff --git a/src/matrix.ts b/src/matrix.ts index 0c83824..3a9f723 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -28,6 +28,14 @@ class MatrixHandler_ { ) { if (this.handledTransactionIDs.indexOf(txnId) > -1) return; + let matrixClient; + try { + matrixClient = await Matrix.for("_discord_bot"); + } catch (e) { + console.error("Failed to get main bridge account", e); + return; + } + for (const event of ephemeral) { console.log("[e:Matrix->]", event); @@ -77,8 +85,14 @@ class MatrixHandler_ { } break; } - default: + default: { console.warn("Unhandled event (unknown type)", event); + + await matrixClient.replyNotice( + event as any, + "**Can't bridge** Unknown event type: " + event.type + ); + } } } } @@ -336,7 +350,10 @@ class MatrixHandler_ { default: { console.warn("Unhandled new room message type", message); - await matrixClient.replyNotice(message); + await matrixClient.replyNotice( + message, + "**Can't bridge** Unknown message type: " + message.content.msgtype + ); } } } -- GitLab From b64461518979f65be7c04c50ba649b256af13539 Mon Sep 17 00:00:00 2001 From: Grant Date: Tue, 26 Nov 2024 22:06:05 -0700 Subject: [PATCH 18/21] unintentionally committed --- src/matrix.ts | 58 +++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/matrix.ts b/src/matrix.ts index 3a9f723..d3975aa 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -279,35 +279,35 @@ class MatrixHandler_ { const profile = await Matrix.getUserProfile(message.sender); switch (message.content.msgtype) { - // case "m.text": { - // for (const room of bridged) { - // try { - // const discMsg = await Discord.sendMessage( - // room.discord_guild_id, - // room.discord_channel_id, - // profile, - // Discord.matrixToDiscord(message), - // { - // replyTo: inReplyTo, - // } - // ); - - // await prisma.matrixMessages.create({ - // data: { - // matrix_event_id: message.event_id, - // matrix_room_id: message.room_id, - // matrix_author_id: message.sender, - // discord_id: discMsg.id, - // discord_channel_id: discMsg.channelId, - // discord_guild_id: discMsg.guildId!, - // }, - // }); - // } catch (e) { - // console.error("Failed to bridge to room", room, e); - // } - // } - // break; - // } + case "m.text": { + for (const room of bridged) { + try { + const discMsg = await Discord.sendMessage( + room.discord_guild_id, + room.discord_channel_id, + profile, + Discord.matrixToDiscord(message), + { + replyTo: inReplyTo, + } + ); + + await prisma.matrixMessages.create({ + data: { + matrix_event_id: message.event_id, + matrix_room_id: message.room_id, + matrix_author_id: message.sender, + discord_id: discMsg.id, + discord_channel_id: discMsg.channelId, + discord_guild_id: discMsg.guildId!, + }, + }); + } catch (e) { + console.error("Failed to bridge to room", room, e); + } + } + break; + } case "m.file": case "m.video": case "m.audio": -- GitLab From 7d4e3a87c5901a4bb052e2c6068984beb5b9b9b3 Mon Sep 17 00:00:00 2001 From: Grant Date: Tue, 26 Nov 2024 22:37:17 -0700 Subject: [PATCH 19/21] handle unknown message types for discord -> matrix --- src/discord.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/discord.ts b/src/discord.ts index 19e9e56..ce81845 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -16,6 +16,7 @@ import { Message, MessageCreateOptions, MessagePayload, + MessageType, PartialMessage, Partials, Typing, @@ -87,6 +88,11 @@ class Discord_ { return message.attachments.get(attachment_id)?.url; } + readonly ALLOWED_MESSAGE_TYPES: MessageType[] = [ + MessageType.Default, + MessageType.Reply, + ]; + async handleMessageCreate(message: Message) { if (message.author.id === this.client.user!.id) return; // ignore bot's own messages if (message.webhookId) return; // if it's a webhook we don't care about it @@ -98,6 +104,20 @@ class Discord_ { }); if (!bridged.length) return; // no valid bridges + if (message.poll) { + await message.reply( + "**Cannot be bridged** Discord polls cannot be bridged" + ); + return; + } + + if (this.ALLOWED_MESSAGE_TYPES.indexOf(message.type) === -1) { + await message.reply( + `**Cannot be bridged** Message type cannot be bridged (${message.type})` + ); + return; + } + var content = message.content; content = await this.replaceRoleMentions(message.guildId, content); -- GitLab From a76b40864f63c120a81695512060490bb9e62d41 Mon Sep 17 00:00:00 2001 From: Grant Date: Wed, 27 Nov 2024 00:15:47 -0700 Subject: [PATCH 20/21] matrix -> discord stickers --- src/matrix.ts | 47 +++++++++++++++++++++++++++++++++++++++++++++ src/types/matrix.ts | 8 ++++++++ 2 files changed, 55 insertions(+) diff --git a/src/matrix.ts b/src/matrix.ts index d3975aa..617c38b 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -10,6 +10,7 @@ import { MatrixRoomMembership, MatrixRoomMessage, MatrixRoomRedaction, + MatrixStickerEvent, } from "./types/matrix"; /** @@ -85,6 +86,14 @@ class MatrixHandler_ { } break; } + case "m.sticker": { + if (event.sender.startsWith("@_discord")) continue; + const event2: MatrixStickerEvent = event as any; + + // these messages cannot be edited + await this.handleSticker(event2); + break; + } default: { console.warn("Unhandled event (unknown type)", event); @@ -125,6 +134,44 @@ class MatrixHandler_ { return { status: 200 }; } + async handleSticker(event: MatrixStickerEvent) { + const bridges = await Glue.getBridges({ matrix_room: event.room_id }); + + const profile = await Matrix.getUserProfile(event.sender); + + for (const bridge of bridges) { + try { + const discMsg = await Discord.sendMessage( + bridge.discord_guild_id, + bridge.discord_channel_id, + profile, + "", + { + attachment: { + url: Matrix.getMediaURL(event.content.url, "internal") as string, + name: + "sticker." + + (event.content.info.mimetype?.split("/").at(-1) || "png"), + }, + } + ); + + await prisma.matrixMessages.create({ + data: { + matrix_event_id: event.event_id, + matrix_room_id: event.room_id, + matrix_author_id: event.sender, + discord_id: discMsg.id, + discord_channel_id: discMsg.channelId, + discord_guild_id: discMsg.guildId!, + }, + }); + } catch (e) { + console.error("Failed to bridge message", e); + } + } + } + /** * Handle m.typing messages * @param event diff --git a/src/types/matrix.ts b/src/types/matrix.ts index 0265d2f..57e79d8 100644 --- a/src/types/matrix.ts +++ b/src/types/matrix.ts @@ -24,6 +24,14 @@ export interface IClientEvent { unsigned?: MatrixUnsignedData; } +export type MatrixStickerEvent = { type: "m.sticker" } & IClientEvent<{ + body: string; + info: ImageInfo & { + thumbnail_info?: Omit; + }; + url: MXC; +}>; + export interface IEphemeralEvent< Type extends string = string, Content = unknown, -- GitLab From 717abb1136352ce90bb1bc02ee45964096764d63 Mon Sep 17 00:00:00 2001 From: Grant <3380410-grahhnt@users.noreply.gitlab.com> Date: Wed, 27 Nov 2024 12:42:55 -0700 Subject: [PATCH 21/21] discord -> matrix stickers --- .../migration.sql | 2 + prisma/schema.prisma | 1 + src/discord.ts | 62 +++++++++++++++++++ src/webserver.ts | 13 ++++ 4 files changed, 78 insertions(+) create mode 100644 prisma/migrations/20241127192733_add_sticker_id_to_discord_messages/migration.sql diff --git a/prisma/migrations/20241127192733_add_sticker_id_to_discord_messages/migration.sql b/prisma/migrations/20241127192733_add_sticker_id_to_discord_messages/migration.sql new file mode 100644 index 0000000..6ac7e43 --- /dev/null +++ b/prisma/migrations/20241127192733_add_sticker_id_to_discord_messages/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "DiscordMessages" ADD COLUMN "sticker_id" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 37c7c0f..6c13778 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -33,6 +33,7 @@ model DiscordMessages { discord_guild_id String discord_author_id String attachment_id String? + sticker_id String? matrix_event_id String matrix_room_id String diff --git a/src/discord.ts b/src/discord.ts index ce81845..b7da532 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -28,8 +28,10 @@ import { import { prisma } from "./lib/prisma"; import { IMatrixUser, + MXC, MatrixRoomMessage, MatrixRoomRedaction, + MatrixStickerEvent, } from "./types/matrix"; import { Matrix } from "./lib/matrix"; import { FriendlyError } from "./lib/utils"; @@ -65,6 +67,29 @@ class Discord_ { this.client.login(token); } + getBridgeProxy( + type: "attachment", + ...args: [ + guild_id: string, + channel_id: string, + message_id: string, + attachment_id: string, + ] + ): MXC; + getBridgeProxy(type: "avatar", ...args: [user_id: string]): MXC; + getBridgeProxy(type: "sticker", ...args: [sticker_id: string]): MXC; + getBridgeProxy(type: string, ...args: never[]): MXC { + const mxcBase = `mxc://${process.env.HOST}` as const; + switch (type) { + case "attachment": + case "avatar": + case "sticker": + return `${mxcBase}/${Buffer.from([type, ...args].join(",")).toString("base64url")}`; + } + + throw new Error("Unknown proxy type"); + } + async getAttachmentURL( guild_id: string, channel_id: string, @@ -195,6 +220,43 @@ class Discord_ { } }); + for (const [id, sticker] of message.stickers) { + await matrixClient + .sendEvent( + room.matrix_room_id, + { + type: "m.sticker", + content: { + body: sticker.name, + url: this.getBridgeProxy("sticker", id), + info: { + w: 180, + h: 180, + mimetype: "image/png", + }, + }, + }, + [room.matrix_room_id, message.id, id].join(",") + ) + .then((result) => { + if (result.status === 200) { + return prisma.discordMessages.create({ + data: { + discord_id: message.id, + discord_guild_id: message.guildId!, + discord_channel_id: message.channelId, + discord_author_id: message.author.id, + attachment_id: id, + matrix_event_id: result.data.event_id, + matrix_room_id: room.matrix_room_id, + }, + }); + } else { + console.error("Failed to send message", result); + } + }); + } + for (const [id, attachment] of message.attachments) { let attachmentType: MatrixRoomMessage["content"]["msgtype"] = "m.file"; diff --git a/src/webserver.ts b/src/webserver.ts index d5780ae..0331dc0 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -230,6 +230,19 @@ class Webserver_ { targetURL = avatarURL; break; } + case "sticker": { + const [stickerid] = parts; + + const sticker = await Discord.client + .fetchSticker(stickerid) + .catch((e) => null); + if (!sticker) { + return res.status(404).json({}); + } + + targetURL = sticker.url; + break; + } default: return res .status(404) -- GitLab