Merge branch 'hs/update-m-a-b' into release-v0.21.0

app.js

@@ -5,6 +5,7 @@ const main = require("./lib/main");
 const path = require("path");
 
 const REG_PATH = "appservice-registration-irc.yaml";
+let bridge;
 new Cli({
     registrationPath: REG_PATH,
     enableRegistration: true,
@@ -33,7 +34,10 @@ new Cli({
                     requestTimeoutSeconds: 60 * 5
                 }
             }
-        }
+        },
+    },
+    onConfigChanged: function(config) {
+        bridge.onConfigChanged(config);
     },
     generateRegistration: function(reg, callback) {
         try {
@@ -49,7 +53,9 @@ new Cli({
         if (port === -1) {
             port = null;
         }
-        const bridge = main.runBridge(port, config, reg).catch(function(err) {
+        main.runBridge(port, config, reg).then((resultBridge) => {
+            bridge = resultBridge;
+        }).catch(function(err) {
             log.error("Failed to run bridge.");
             log.error(err);
             process.exit(1);
@@ -58,7 +64,7 @@ new Cli({
         process.on("SIGTERM", async () => {
             log.info("SIGTERM recieved, killing bridge");
             try {
-                await main.killBridge(await bridge);
+                await main.killBridge(bridge);
             }
             catch (ex) {
                 log.error("Failed to killBridge");

changelog.d/1145.feature

@@ -0,0 +1 @@
+Add support for reconfiguring the bridge at runtime by sending a `SIGHUP`

changelog.d/1146.bugfix

@@ -0,0 +1 @@
+Fix more cases of double bridged users

changelog.d/1148.misc

@@ -0,0 +1 @@
+Add index to client_config for `config->>username` to speed up username lookups

changelog.d/1151.bugfix

@@ -0,0 +1 @@
+Fix a bug where a user leaving with a reason would cause them to join then leave

config.sample.yaml

@@ -479,10 +479,6 @@ ircService:
     eventCacheSize: 4096
 
   ircHandler:
-    # How many /leave requests can be ongoing at a time.
-    # This is used to stem the flow of requests in case of a mass quit/leave, which might
-    # slow down the homeserver.
-    leaveConcurrency: 10
     # Should we attempt to match an IRC side mention (nickaname match)
     # with the nickname's owner's matrixId, if we are bridging them?
     # "on" - Defaults to enabled, users can choose to disable.

config.schema.yml

@@ -124,8 +124,6 @@ properties:
             ircHandler:
                 type: "object"
                 properties:
-                    leaveConcurrency:
-                        type: "integer"
                     mapIrcMentionsToMatrix:
                         type: "string"
                         enum: ["on", "off", "force-off"]

package.json

@@ -35,8 +35,8 @@
     "irc": "matrix-org/node-irc#9028c2197c216dd8e6fc2cb3cc07ce2d6bf741a7",
     "js-yaml": "^3.2.7",
     "logform": "^2.1.2",
-    "matrix-appservice": "^0.5.0",
-    "matrix-appservice-bridge": "2.0.0-rc1",
+    "matrix-appservice": "^0.6.0",
+    "matrix-appservice-bridge": "2.2.0-rc2",
     "matrix-lastactive": "^0.1.5",
     "nedb": "^1.1.2",
     "nopt": "^3.0.1",

spec/integ/invite-rooms.spec.js

@@ -3,44 +3,55 @@ const envBundle = require("../util/env-bundle");
 
 describe("Invite-only rooms", function() {
     const {env, config, roomMapping, botUserId, test} = envBundle();
-    let testUser = {
+    const testUser = {
         id: "@flibble:wibble",
         nick: "flibble"
     };
-    let testIrcUser = {
+    const testIrcUser = {
         localpart: roomMapping.server + "_foobar",
         id: "@" + roomMapping.server + "_foobar:" + config.homeserver.domain,
         nick: "foobar"
     };
 
 
-    beforeEach(test.coroutine(function*() {
-        yield test.beforeEach(env);
+    beforeEach(async () => {
+        await test.beforeEach(env);
 
         env.ircMock._autoConnectNetworks(
             roomMapping.server, roomMapping.botNick, roomMapping.server
         );
 
         // do the init
-        yield test.initEnv(env);
-    }));
+        await test.initEnv(env);
+    });
 
-    afterEach(test.coroutine(function*() {
-        yield test.afterEach(env);
-    }));
+    afterEach(async () => {
+        await test.afterEach(env);
+    });
 
-    it("should be joined by the bot if the AS does know the room ID",
-    function(done) {
-        let adminRoomId = "!adminroom:id";
-        let sdk = env.clientMock._client(botUserId);
+    it("should be joined by the bot if the AS does know the room ID", async() => {
+        const adminRoomId = "!adminroom:id";
+        const sdk = env.clientMock._client(botUserId);
         let joinRoomCount = 0;
-        sdk.joinRoom.and.callFake(function(roomId) {
+        sdk.joinRoom.and.callFake(async (roomId) => {
             expect(roomId).toEqual(adminRoomId);
             joinRoomCount += 1;
-            return Promise.resolve({});
+            return {room_id: roomId};
         });
 
-        env.mockAppService._trigger("type:m.room.member", {
+        await env.mockAppService._trigger("type:m.room.member", {
+            content: {
+                membership: "invite",
+                is_direct: true,
+            },
+            state_key: botUserId,
+            user_id: testUser.id,
+            room_id: adminRoomId,
+            type: "m.room.member"
+        });
+        expect(joinRoomCount).toEqual(1, "Failed to join admin room");
+        // inviting them AGAIN to an existing known ADMIN room should trigger a join
+        await env.mockAppService._trigger("type:m.room.member", {
             content: {
                 membership: "invite",
                 is_direct: true,
@@ -49,26 +60,8 @@ describe("Invite-only rooms", function() {
             user_id: testUser.id,
             room_id: adminRoomId,
             type: "m.room.member"
-        }).then(function() {
-            expect(joinRoomCount).toEqual(1, "Failed to join admin room");
-            // inviting them AGAIN to an existing known ADMIN room should trigger a join
-            return env.mockAppService._trigger("type:m.room.member", {
-                content: {
-                    membership: "invite",
-                    is_direct: true,
-                },
-                state_key: botUserId,
-                user_id: testUser.id,
-                room_id: adminRoomId,
-                type: "m.room.member"
-            });
-        }).then(function() {
-            expect(joinRoomCount).toEqual(2, "Failed to join admin room again");
-            done();
-        }, function(err) {
-            expect(true).toBe(false, "Failed to join admin room again: " + err);
-            done();
         });
+        expect(joinRoomCount).toEqual(2, "Failed to join admin room again");
     });
 
     it("should be joined by a virtual IRC user if the bot invited them, " +

src/bridge/IrcBridge.ts

@@ -184,7 +184,58 @@ export class IrcBridge {
             homeserverToken,
             httpMaxSizeBytes: (this.config.advanced || { }).maxTxnSize || TXN_SIZE_DEFAULT,
         });
+    }
+
+    public async onConfigChanged(newConfig: BridgeConfig) {
+        log.info(`Bridge config was reloaded, applying changes`);
+        const oldConfig = this.config;
+
+        if (oldConfig.advanced.maxHttpSockets !== newConfig.advanced.maxHttpSockets) {
+            const maxSockets = (newConfig.advanced || {maxHttpSockets: 1000}).maxHttpSockets;
+            require("http").globalAgent.maxSockets = maxSockets;
+            require("https").globalAgent.maxSockets = maxSockets;
+            log.info(`Adjusted max sockets to ${maxSockets}`);
+        }
+
+        // We can't modify the maximum payload size after starting the http listener for the bridge, so
+        // newConfig.advanced.maxTxnSize is ignored.
+
+        if (oldConfig.homeserver.dropMatrixMessagesAfterSecs !== newConfig.homeserver.dropMatrixMessagesAfterSecs) {
+            oldConfig.homeserver.dropMatrixMessagesAfterSecs = newConfig.homeserver.dropMatrixMessagesAfterSecs;
+            log.info(`Adjusted dropMatrixMessagesAfterSecs to ${newConfig.homeserver.dropMatrixMessagesAfterSecs}`);
+        }
+
+        if (oldConfig.homeserver.media_url !== newConfig.homeserver.media_url) {
+            oldConfig.homeserver.media_url = newConfig.homeserver.media_url;
+            log.info(`Adjusted media_url to ${newConfig.homeserver.media_url}`);
+        }
+
+        this.ircHandler.onConfigChanged(newConfig.ircHandler || {});
+
+        const hasLoggingChanged = JSON.stringify(oldConfig.ircService.logging)
+            !== JSON.stringify(newConfig.ircService.logging);
+        if (hasLoggingChanged) {
+            Logging.configure(newConfig.ircService.logging);
+        }
 
+        await this.dataStore.removeConfigMappings();
+
+        // All config mapped channels will be briefly unavailable
+        await Promise.all(this.ircServers.map(async (server) => {
+            let newServerConfig = newConfig.ircService.servers[server.domain];
+            if (!newServerConfig) {
+                log.warn(`Server ${server.domain} removed from config. Bridge will need to be restarted`);
+                return;
+            }
+            newServerConfig = extend(
+                true, {}, IrcServer.DEFAULT_CONFIG, newConfig.ircService.servers[server.domain]
+            );
+            server.reconfigure(newServerConfig, newConfig.homeserver.dropMatrixMessagesAfterSecs);
+            await this.dataStore.setServerFromConfig(server, newServerConfig);
+        }));
+
+        await this.fetchJoinedRooms();
+        await this.joinMappedMatrixRooms();
     }
 
     private initialiseMetrics(bindPort: number) {
@@ -499,16 +550,12 @@ export class IrcBridge {
         const allUsers = await this.dataStore.getAllUserIds();
         const bot = this.bridge.getBot();
         allUsers.filter((u) => bot.isRemoteUser(u))
-            .forEach((u) => this.membershipCache.setMemberEntry("", u, "join"));
+            .forEach((u) => this.membershipCache.setMemberEntry("", u, "join", {}));
 
 
         log.info("Fetching Matrix rooms that are already joined to...");
         await this.fetchJoinedRooms();
 
-        for (const roomId of this.joinedRoomList) {
-            this.membershipCache.setMemberEntry(roomId, this.appServiceUserId, "join");
-        }
-
         if (this.config.ircService.bridgeInfoState?.enabled) {
             this.bridgeStateSyncer = new BridgeStateSyncer(this.dataStore, this.bridge, this);
             if (this.config.ircService.bridgeInfoState.initial) {

src/bridge/IrcHandler.ts

@@ -37,7 +37,6 @@ interface TopicQueueItem {
 
 export interface IrcHandlerConfig {
     mapIrcMentionsToMatrix?: "on"|"off"|"force-off";
-    leaveConcurrency?: number;
 }
 
 type MetricNames = "join.names"|"join"|"part"|"pm"|"invite"|"topic"|"message"|"kick"|"mode";
@@ -69,7 +68,7 @@ export class IrcHandler {
     "off" - Defaults to disabled, users can choose to enable.
     "force-off" - Disabled, cannot be enabled.
     */
-    private readonly mentionMode: "on"|"off"|"force-off";
+    private mentionMode: "on"|"off"|"force-off";
 
     public readonly roomAccessSyncer: RoomAccessSyncer;
 
@@ -956,6 +955,10 @@ export class IrcHandler {
         return metrics;
     }
 
+    public onConfigChanged(config: IrcHandlerConfig) {
+        this.mentionMode = config.mapIrcMentionsToMatrix || "on";
+    }
+
     private invalidateNickUserIdMap(server: IrcServer, channel: string) {
         this.nickUserIdMapCache.delete(`${server.domain}:${channel}`);
     }

src/bridge/MemberListSyncer.ts

@@ -195,7 +195,7 @@ export class MemberListSyncer {
             // fetch joined members allowing 50 in-flight reqs at a time
             const pool = new QueuePool(50, async (_roomId) => {
                 const roomId = _roomId as string;
-                let userMap: Record<string, {display_name: string; avatar: string}>|undefined;
+                let userMap: Record<string, {display_name: string}>|undefined;
                 while (!userMap) {
                     try {
                         userMap = await this.appServiceBot.getJoinedMembers(roomId);

src/config/BridgeConfig.ts

@@ -6,7 +6,7 @@ export interface BridgeConfig {
     matrixHandler: {
 
     };
-    ircHandler: IrcHandlerConfig;
+    ircHandler?: IrcHandlerConfig;
     database: {
         engine: string;
         connectionString: string;

src/datastore/postgres/PgDataStore.ts

@@ -33,7 +33,7 @@ const log = getLogger("PgDatastore");
 export class PgDataStore implements DataStore {
     private serverMappings: {[domain: string]: IrcServer} = {};
 
-    public static readonly LATEST_SCHEMA = 4;
+    public static readonly LATEST_SCHEMA = 5;
     private pgPool: Pool;
     private hasEnded = false;
     private cryptoStore?: StringCrypto;