From 49ea2def1b422c538727ac6023c7e0d2fd1a195d Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Wed, 23 Sep 2020 01:17:13 +0200 Subject: [PATCH 1/5] Fix eslint warnings --- src/datastore/postgres/PgDatastore.ts | 28 +++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/datastore/postgres/PgDatastore.ts b/src/datastore/postgres/PgDatastore.ts index e08514cd3..53f7a6a7c 100644 --- a/src/datastore/postgres/PgDatastore.ts +++ b/src/datastore/postgres/PgDatastore.ts @@ -43,8 +43,7 @@ const log = Logging.get("PgDatastore"); export class PgDatastore implements Datastore { public static readonly LATEST_SCHEMA = 9; - // tslint:disable-next-line: no-any - public readonly postgresDb: IDatabase; + public readonly postgresDb: IDatabase; constructor(connectionString: string) { this.postgresDb = pgp(connectionString); @@ -120,7 +119,13 @@ export class PgDatastore implements Datastore { return this.postgresDb.none("DELETE FROM linked_accounts WHERE slack_id = ${slackId} AND user_id = ${userId}", { userId, slackId }); } - public async upsertEvent(roomIdOrEntry: string|EventEntry, eventId?: string, channelId?: string, ts?: string, extras?: EventEntryExtra) { + public async upsertEvent( + roomIdOrEntry: string|EventEntry, + eventId?: string, + channelId?: string, + ts?: string, + extras?: EventEntryExtra + ): Promise { let entry: EventEntry = roomIdOrEntry as EventEntry; if (typeof(roomIdOrEntry) === "string") { entry = { @@ -233,7 +238,7 @@ export class PgDatastore implements Datastore { ); } - public async ensureSchema() { + public async ensureSchema(): Promise { let currentVersion = await this.getSchemaVersion(); while (currentVersion < PgDatastore.LATEST_SCHEMA) { log.info(`Updating schema to v${currentVersion + 1}`); @@ -251,7 +256,7 @@ export class PgDatastore implements Datastore { log.info(`Database schema is at version v${currentVersion}`); } - public async upsertRoom(room: BridgedRoom) { + public async upsertRoom(room: BridgedRoom): Promise { const entry = room.toEntry(); log.debug(`upsertRoom: ${entry.id}`); return this.postgresDb.none( @@ -265,12 +270,12 @@ export class PgDatastore implements Datastore { ); } - public async deleteRoom(id: string) { + public async deleteRoom(id: string): Promise { log.debug(`deleteRoom: ${id}`); return this.postgresDb.none("DELETE FROM rooms WHERE id = ${id}", { id }); } - public async getAllRooms() { + public async getAllRooms(): Promise { const entries = await this.postgresDb.manyOrNone("SELECT * FROM rooms"); return entries.map((r) => { const remote = JSON.parse(r.json); @@ -283,7 +288,7 @@ export class PgDatastore implements Datastore { }); } - public async upsertTeam(entry: TeamEntry) { + public async upsertTeam(entry: TeamEntry): Promise { log.debug(`upsertTeam: ${entry.id} ${entry.name}`); const props = { id: entry.id, @@ -296,11 +301,10 @@ export class PgDatastore implements Datastore { user_id: entry.user_id, }; const statement = PgDatastore.BuildUpsertStatement("teams", ["id"], [props]); - await this.postgresDb.none(statement, props); + return this.postgresDb.none(statement, props); } - // tslint:disable-next-line: no-any - private static teamEntryForRow(doc: any) { + private static teamEntryForRow(doc: Record): TeamEntry { return { id: doc.id, name: doc.name, @@ -339,7 +343,7 @@ export class PgDatastore implements Datastore { ); } - public async removePuppetTokenByMatrixId(teamId: string, matrixId: string) { + public async removePuppetTokenByMatrixId(teamId: string, matrixId: string): Promise { return this.postgresDb.none("DELETE FROM puppets WHERE slackteam = ${teamId} " + "AND matrixuser = ${matrixId}", { teamId, matrixId }); } From d035bbd6b5dc223f6d1a1c4be4fe779829db00a8 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Wed, 23 Sep 2020 01:37:46 +0200 Subject: [PATCH 2/5] Add database for custom emojis --- src/datastore/Models.ts | 5 +++++ src/datastore/NedbDatastore.ts | 15 +++++++++++++ src/datastore/postgres/PgDatastore.ts | 32 ++++++++++++++++++++++++++- src/datastore/postgres/schema/v10.ts | 13 +++++++++++ src/tests/utils/fakeDatastore.ts | 12 ++++++++++ 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/datastore/postgres/schema/v10.ts diff --git a/src/datastore/Models.ts b/src/datastore/Models.ts index 9cf2a5079..2635fd65d 100644 --- a/src/datastore/Models.ts +++ b/src/datastore/Models.ts @@ -109,6 +109,11 @@ export interface Datastore { deleteRoom(id: string): Promise; getAllRooms(): Promise; + // Custom emojis + upsertEmoji(teamId: string, name: string, mxc: string): Promise; + getEmojiMxc(teamId: string, name: string): Promise; + deleteEmoji(teamId: string, name: string): Promise; + // Events upsertEvent(roomId: string, eventId: string, channelId: string, ts: string, extras?: EventEntryExtra): Promise; upsertEvent(roomIdOrEntry: EventEntry): Promise; diff --git a/src/datastore/NedbDatastore.ts b/src/datastore/NedbDatastore.ts index 6936f4bf9..9f88faf72 100644 --- a/src/datastore/NedbDatastore.ts +++ b/src/datastore/NedbDatastore.ts @@ -183,6 +183,21 @@ export class NedbDatastore implements Datastore { }); } + public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { + // no-op; custom emoji are not implemented for NeDB + return null; + } + + public async getEmojiMxc(teamId: string, name: string): Promise { + // no-op; custom emoji are not implemented for NeDB + return null; + } + + public async deleteEmoji(teamId: string, name: string): Promise { + // no-op; custom emoji are not implemented for NeDB + return null; + } + public async upsertEvent(roomIdOrEntry: string|EventEntry, eventId?: string, channelId?: string, ts?: string, extras?: EventEntryExtra): Promise { let storeEv: StoredEvent; diff --git a/src/datastore/postgres/PgDatastore.ts b/src/datastore/postgres/PgDatastore.ts index 53f7a6a7c..502d8f346 100644 --- a/src/datastore/postgres/PgDatastore.ts +++ b/src/datastore/postgres/PgDatastore.ts @@ -42,7 +42,7 @@ const pgp: IMain = pgInit({ const log = Logging.get("PgDatastore"); export class PgDatastore implements Datastore { - public static readonly LATEST_SCHEMA = 9; + public static readonly LATEST_SCHEMA = 10; public readonly postgresDb: IDatabase; constructor(connectionString: string) { @@ -119,6 +119,36 @@ export class PgDatastore implements Datastore { return this.postgresDb.none("DELETE FROM linked_accounts WHERE slack_id = ${slackId} AND user_id = ${userId}", { userId, slackId }); } + public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { + log.debug(`upsertEmoji: ${teamId} ${name} ${mxc}`); + return this.postgresDb.none( + "INSERT INTO emojis(slack_team_id, name, mxc) " + + "VALUES(${teamId}, ${name}, ${mxc})" + + "ON CONFLICT ON CONSTRAINT emojis_slack_idx DO UPDATE SET mxc = ${mxc}", + { + teamId, + name, + mxc, + }, + ); + } + + public async getEmojiMxc(teamId: string, name: string): Promise { + log.debug(`upsertEmoji: ${teamId} ${name}`); + // TODO Resolve aliases + return this.postgresDb.oneOrNone( + "SELECT mxc FROM emojis WHERE team_id = ${teamId} AND name = ${name}", + { teamId, name }, + a => a.mxc, + ); + } + + public async deleteEmoji(teamId: string, name: string): Promise { + log.debug(`deleteEmoji: ${teamId} ${name}`); + // TODO Delete aliases + return this.postgresDb.none("DELETE FROM emojis WHERE slack_team_id = ${teamId} AND name = ${name}", { teamId, name }); + } + public async upsertEvent( roomIdOrEntry: string|EventEntry, eventId?: string, diff --git a/src/datastore/postgres/schema/v10.ts b/src/datastore/postgres/schema/v10.ts new file mode 100644 index 000000000..7d3cc72ab --- /dev/null +++ b/src/datastore/postgres/schema/v10.ts @@ -0,0 +1,13 @@ +import { IDatabase } from "pg-promise"; + +// tslint:disable-next-line: no-any +export const runSchema = async(db: IDatabase) => { + await db.none(` + CREATE TABLE emojis ( + slack_team_id TEXT NOT NULL, + name TEXT NOT NULL, + mxc TEXT NOT NULL + ); + CREATE UNIQUE INDEX emojis_slack_idx ON emojis (slack_team_id, name); + `); +}; diff --git a/src/tests/utils/fakeDatastore.ts b/src/tests/utils/fakeDatastore.ts index 61154b6a5..6da30af4f 100644 --- a/src/tests/utils/fakeDatastore.ts +++ b/src/tests/utils/fakeDatastore.ts @@ -72,6 +72,18 @@ export class FakeDatastore implements Datastore { throw Error("Method not implemented."); } + public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { + throw Error("Method not implemented."); + } + + public async getEmojiMxc(teamId: string, name: string): Promise { + throw Error("Method not implemented."); + } + + public async deleteEmoji(teamId: string, name: string): Promise { + throw Error("Method not implemented."); + } + public async upsertEvent(roomId: string, eventId: string, channelId: string, ts: string, extras?: EventEntryExtra): Promise; public async upsertEvent(roomIdOrEntry: EventEntry): Promise; From 6d766b177264ebbbecafe08876d563d34ad77021 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Wed, 23 Sep 2020 16:33:27 +0200 Subject: [PATCH 3/5] Sync custom emoji and handle changes --- changelog.d/500.feature | 1 + src/SlackEventHandler.ts | 47 +++++++++++++++++++++++++- src/TeamSyncer.ts | 48 +++++++++++++++++++++++++-- src/datastore/Models.ts | 8 ++--- src/datastore/NedbDatastore.ts | 6 ++-- src/datastore/postgres/PgDatastore.ts | 21 ++++++------ src/datastore/postgres/schema/v10.ts | 4 +-- src/tests/utils/fakeDatastore.ts | 6 ++-- 8 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 changelog.d/500.feature diff --git a/changelog.d/500.feature b/changelog.d/500.feature new file mode 100644 index 000000000..da3379077 --- /dev/null +++ b/changelog.d/500.feature @@ -0,0 +1 @@ +Support custom Slack emoji \ No newline at end of file diff --git a/src/SlackEventHandler.ts b/src/SlackEventHandler.ts index b08992014..dd77e0935 100644 --- a/src/SlackEventHandler.ts +++ b/src/SlackEventHandler.ts @@ -29,6 +29,17 @@ interface ISlackEventChannelRename extends ISlackEvent { created: number; } +/** + * https://api.slack.com/events/emoji_changed + */ +interface ISlackEventEmojiChanged extends ISlackEvent { + event_ts: string; + subtype?: "add"|"remove"|unknown; + name?: string; + names?: string[]; + value?: string; +} + /** * https://api.slack.com/events/team_domain_change */ @@ -102,7 +113,7 @@ export class SlackEventHandler extends BaseSlackHandler { */ protected static SUPPORTED_EVENTS: string[] = ["message", "reaction_added", "reaction_removed", "team_domain_change", "channel_rename", "user_change", "user_typing", "member_joined_channel", - "channel_created", "channel_deleted", "team_join"]; + "channel_created", "channel_deleted", "team_join", "emoji_changed"]; constructor(main: Main) { super(main); } @@ -185,6 +196,8 @@ export class SlackEventHandler extends BaseSlackHandler { case "reaction_removed": await this.handleReaction(event as ISlackEventReaction, teamId); break; + case "emoji_changed": + await this.handleEmojiChangedEvent(event as ISlackEventEmojiChanged, teamId); case "channel_rename": await this.handleChannelRenameEvent(event as ISlackEventChannelRename); break; @@ -335,6 +348,38 @@ export class SlackEventHandler extends BaseSlackHandler { } } + private async handleEmojiChangedEvent(event: ISlackEventEmojiChanged, teamId: string) { + if (!this.main.teamSyncer) { + throw Error("ignored"); + } + switch(event.subtype) { + case "add": { + if (typeof event.name !== 'string') { + throw Error('Slack event emoji_changed is expected to have name: string'); + } + if (typeof event.value !== 'string' || !/^https:\/\/|alias:/.test(event.value)) { + throw Error('Slack event emoji_changed is expected to have value: string and start with "https://" or "alias:"'); + } + const client = await this.main.clientFactory.getTeamClient(teamId); + await this.main.teamSyncer.addCustomEmoji(teamId, event.name, event.value, client.token!); + return; + } + case "remove": + if (!Array.isArray(event.names) || event.names.some(v => typeof v !== 'string')) { + throw Error('Slack event emoji_changed is expected to have names: string[]'); + } + for (const name of event.names) { + await this.main.teamSyncer.removeCustomEmoji(teamId, name); + } + break; + default: { + const client = await this.main.clientFactory.getTeamClient(teamId); + await this.main.teamSyncer.syncCustomEmoji(teamId, client); + break; + } + } + } + private async handleDomainChangeEvent(event: ISlackEventTeamDomainChange, teamId: string) { const team = await this.main.datastore.getTeam(teamId); if (team) { diff --git a/src/TeamSyncer.ts b/src/TeamSyncer.ts index 802151052..8095789de 100644 --- a/src/TeamSyncer.ts +++ b/src/TeamSyncer.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import axios from "axios"; import { Logging } from "matrix-appservice-bridge"; import { BridgedRoom } from "./BridgedRoom"; import { Main } from "./Main"; @@ -59,7 +60,7 @@ export class TeamSyncer { this.teamConfigs = config.team_sync; } - public async syncAllTeams(teamClients: { [id: string]: WebClient; }) { + public async syncAllTeams(teamClients: { [id: string]: WebClient; }): Promise { const queue = new PQueue({concurrency: TEAM_SYNC_CONCURRENCY}); const functionsForQueue: (() => Promise)[] = []; for (const [teamId, client] of Object.entries(teamClients)) { @@ -71,6 +72,7 @@ export class TeamSyncer { log.info("Syncing team", teamId); await this.syncItems(teamId, client, "user"); await this.syncItems(teamId, client, "channel"); + await this.syncCustomEmoji(teamId, client); }); } try { @@ -88,7 +90,6 @@ export class TeamSyncer { log.warn(`Not syncing ${type}s for ${teamId}`); return; } - // tslint:disable-next-line: no-any let itemList: any[] = []; let cursor: string|undefined; for (let i = 0; i < TEAM_SYNC_FAILSAFE && (cursor === undefined || cursor !== ""); i++) { @@ -299,6 +300,49 @@ export class TeamSyncer { } } + public async syncCustomEmoji(teamId: string, client: WebClient): Promise { + // if (!this.getTeamSyncConfig(teamId, 'customEmoji')) { + // log.warn(`Not syncing custom emoji for ${teamId}`); + // return; + // } + log.info(`Syncing custom emoji ${teamId}`); + + const response = await client.emoji.list(); + if (response.ok !== true) { + throw Error("Slack replied to emoji.list but said the response wasn't ok."); + } + if (typeof response.emoji !== "object" || !response.emoji) { + throw Error("Slack replied to emoji.list but the list was not not an object."); + } + for (const [name, url] of Object.values(response.emoji)) { + await this.addCustomEmoji(teamId, name, url, client.token!); + } + } + + public async addCustomEmoji(teamId: string, name: string, url: string, accessToken: string): Promise { + const imageResponse = await axios.get(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + responseType: "arraybuffer", + }); + if (imageResponse.status !== 200) { + throw Error('Failed to get file'); + } + const mxc = await this.main.botIntent.getClient().uploadContent(imageResponse.data, { + name, + type: imageResponse.headers['content-type'], + rawResponse: false, + onlyContentUri: true, + }); + await this.main.datastore.upsertCustomEmoji(teamId, name, mxc); + return mxc; + } + + public async removeCustomEmoji(teamId: string, name: string): Promise { + return this.main.datastore.deleteCustomEmoji(teamId, name); + } + public async onChannelDeleted(teamId: string, channelId: string) { log.info(`${teamId} removed channel ${channelId}`); if (!this.getTeamSyncConfig(teamId, "channel", channelId)) { diff --git a/src/datastore/Models.ts b/src/datastore/Models.ts index 2635fd65d..b5d1e3fc1 100644 --- a/src/datastore/Models.ts +++ b/src/datastore/Models.ts @@ -109,10 +109,10 @@ export interface Datastore { deleteRoom(id: string): Promise; getAllRooms(): Promise; - // Custom emojis - upsertEmoji(teamId: string, name: string, mxc: string): Promise; - getEmojiMxc(teamId: string, name: string): Promise; - deleteEmoji(teamId: string, name: string): Promise; + // Custom emoji + upsertCustomEmoji(teamId: string, name: string, mxc: string): Promise; + getCustomEmojiMxc(teamId: string, name: string): Promise; + deleteCustomEmoji(teamId: string, name: string): Promise; // Events upsertEvent(roomId: string, eventId: string, channelId: string, ts: string, extras?: EventEntryExtra): Promise; diff --git a/src/datastore/NedbDatastore.ts b/src/datastore/NedbDatastore.ts index 9f88faf72..424bb1985 100644 --- a/src/datastore/NedbDatastore.ts +++ b/src/datastore/NedbDatastore.ts @@ -183,17 +183,17 @@ export class NedbDatastore implements Datastore { }); } - public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { + public async upsertCustomEmoji(teamId: string, name: string, mxc: string): Promise { // no-op; custom emoji are not implemented for NeDB return null; } - public async getEmojiMxc(teamId: string, name: string): Promise { + public async getCustomEmojiMxc(teamId: string, name: string): Promise { // no-op; custom emoji are not implemented for NeDB return null; } - public async deleteEmoji(teamId: string, name: string): Promise { + public async deleteCustomEmoji(teamId: string, name: string): Promise { // no-op; custom emoji are not implemented for NeDB return null; } diff --git a/src/datastore/postgres/PgDatastore.ts b/src/datastore/postgres/PgDatastore.ts index 502d8f346..d6830b4ca 100644 --- a/src/datastore/postgres/PgDatastore.ts +++ b/src/datastore/postgres/PgDatastore.ts @@ -119,12 +119,12 @@ export class PgDatastore implements Datastore { return this.postgresDb.none("DELETE FROM linked_accounts WHERE slack_id = ${slackId} AND user_id = ${userId}", { userId, slackId }); } - public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { - log.debug(`upsertEmoji: ${teamId} ${name} ${mxc}`); + public async upsertCustomEmoji(teamId: string, name: string, mxc: string): Promise { + log.debug(`upsertCustomEmoji: ${teamId} ${name} ${mxc}`); return this.postgresDb.none( - "INSERT INTO emojis(slack_team_id, name, mxc) " + + "INSERT INTO custom_emoji(slack_team_id, name, mxc) " + "VALUES(${teamId}, ${name}, ${mxc})" + - "ON CONFLICT ON CONSTRAINT emojis_slack_idx DO UPDATE SET mxc = ${mxc}", + "ON CONFLICT ON CONSTRAINT custom_emoji_slack_idx DO UPDATE SET mxc = ${mxc}", { teamId, name, @@ -133,20 +133,19 @@ export class PgDatastore implements Datastore { ); } - public async getEmojiMxc(teamId: string, name: string): Promise { - log.debug(`upsertEmoji: ${teamId} ${name}`); + public async getCustomEmojiMxc(teamId: string, name: string): Promise { // TODO Resolve aliases return this.postgresDb.oneOrNone( - "SELECT mxc FROM emojis WHERE team_id = ${teamId} AND name = ${name}", + "SELECT mxc FROM custom_emoji WHERE team_id = ${teamId} AND name = ${name}", { teamId, name }, - a => a.mxc, + response => response && response.mxc, ); } - public async deleteEmoji(teamId: string, name: string): Promise { - log.debug(`deleteEmoji: ${teamId} ${name}`); + public async deleteCustomEmoji(teamId: string, name: string): Promise { + log.debug(`deleteCustomEmoji: ${teamId} ${name}`); // TODO Delete aliases - return this.postgresDb.none("DELETE FROM emojis WHERE slack_team_id = ${teamId} AND name = ${name}", { teamId, name }); + return this.postgresDb.none("DELETE FROM custom_emoji WHERE slack_team_id = ${teamId} AND name = ${name}", { teamId, name }); } public async upsertEvent( diff --git a/src/datastore/postgres/schema/v10.ts b/src/datastore/postgres/schema/v10.ts index 7d3cc72ab..a220fe023 100644 --- a/src/datastore/postgres/schema/v10.ts +++ b/src/datastore/postgres/schema/v10.ts @@ -3,11 +3,11 @@ import { IDatabase } from "pg-promise"; // tslint:disable-next-line: no-any export const runSchema = async(db: IDatabase) => { await db.none(` - CREATE TABLE emojis ( + CREATE TABLE custom_emoji ( slack_team_id TEXT NOT NULL, name TEXT NOT NULL, mxc TEXT NOT NULL ); - CREATE UNIQUE INDEX emojis_slack_idx ON emojis (slack_team_id, name); + CREATE UNIQUE INDEX custom_emoji_slack_idx ON custom_emoji (slack_team_id, name); `); }; diff --git a/src/tests/utils/fakeDatastore.ts b/src/tests/utils/fakeDatastore.ts index 6da30af4f..d28addf92 100644 --- a/src/tests/utils/fakeDatastore.ts +++ b/src/tests/utils/fakeDatastore.ts @@ -72,15 +72,15 @@ export class FakeDatastore implements Datastore { throw Error("Method not implemented."); } - public async upsertEmoji(teamId: string, name: string, mxc: string): Promise { + public async upsertCustomEmoji(teamId: string, name: string, mxc: string): Promise { throw Error("Method not implemented."); } - public async getEmojiMxc(teamId: string, name: string): Promise { + public async getCustomEmojiMxc(teamId: string, name: string): Promise { throw Error("Method not implemented."); } - public async deleteEmoji(teamId: string, name: string): Promise { + public async deleteCustomEmoji(teamId: string, name: string): Promise { throw Error("Method not implemented."); } From b31deedff72b5d46e9e1b8bedd44a19b4148a7ee Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Mon, 28 Dec 2020 12:41:12 +0100 Subject: [PATCH 4/5] Docs: Consistently call the registration file slack-registration.yaml --- docs/getting_started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index b29cb89b1..4c218b5da 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -78,7 +78,7 @@ ever stuck, you can post a question in the ```sh $ docker run -v /path/to/config/:/config/ matrixdotorg/matrix-appservice-slack \ - -r -c /config/config.yaml -u "http://$HOST:$MATRIX_PORT" -f /config/slack.yaml + -r -c /config/config.yaml -u "http://$HOST:$MATRIX_PORT" -f /config/slack-registration.yaml ``` 1. Start the actual application service: From 9277087ab6f5807279b9e0d0c36515abd01491c3 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Mon, 4 Jan 2021 20:19:09 +0100 Subject: [PATCH 5/5] Add newsfile --- changelog.d/551.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/551.misc diff --git a/changelog.d/551.misc b/changelog.d/551.misc new file mode 100644 index 000000000..ac84a17d9 --- /dev/null +++ b/changelog.d/551.misc @@ -0,0 +1 @@ +Docs: Consistently call the registration file slack-registration.yaml