From 889895f1d6c146ef40bb1fbefca36ee94d39a374 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 12:56:44 +0100 Subject: [PATCH 001/350] Move lib/ -> src/ --- {lib => src}/DataStore.js | 0 {lib => src}/DebugApi.js | 0 {lib => src}/bridge/IrcBridge.js | 0 {lib => src}/bridge/IrcHandler.js | 0 {lib => src}/bridge/MatrixHandler.js | 0 {lib => src}/bridge/MemberListSyncer.js | 0 {lib => src}/bridge/PublicitySyncer.js | 0 {lib => src}/bridge/QuitDebouncer.js | 0 {lib => src}/bridge/RoomAccessSyncer.js | 0 {lib => src}/config/schema.yml | 0 {lib => src}/config/stats.js | 0 {lib => src}/irc/BridgedClient.js | 0 {lib => src}/irc/ClientPool.js | 0 {lib => src}/irc/ConnectionInstance.js | 0 {lib => src}/irc/IdentGenerator.js | 0 {lib => src}/irc/Ipv6Generator.js | 0 {lib => src}/irc/IrcEventBroker.js | 0 {lib => src}/irc/IrcServer.js | 0 {lib => src}/irc/Scheduler.js | 0 {lib => src}/irc/formatting.js | 0 {lib => src}/irc/ident.js | 0 {lib => src}/logging.js | 0 {lib => src}/main.js | 0 {lib => src}/models/BridgeRequest.js | 0 {lib => src}/models/IrcAction.js | 0 {lib => src}/models/IrcClientConfig.js | 0 {lib => src}/models/IrcRoom.js | 0 {lib => src}/models/IrcUser.js | 0 {lib => src}/models/MatrixAction.js | 0 {lib => src}/promiseutil.js | 0 {lib => src}/provisioning/ProvisionRequest.js | 0 {lib => src}/provisioning/Provisioner.js | 0 {lib => src}/util/Queue.js | 0 {lib => src}/util/QueuePool.js | 0 34 files changed, 0 insertions(+), 0 deletions(-) rename {lib => src}/DataStore.js (100%) rename {lib => src}/DebugApi.js (100%) rename {lib => src}/bridge/IrcBridge.js (100%) rename {lib => src}/bridge/IrcHandler.js (100%) rename {lib => src}/bridge/MatrixHandler.js (100%) rename {lib => src}/bridge/MemberListSyncer.js (100%) rename {lib => src}/bridge/PublicitySyncer.js (100%) rename {lib => src}/bridge/QuitDebouncer.js (100%) rename {lib => src}/bridge/RoomAccessSyncer.js (100%) rename {lib => src}/config/schema.yml (100%) rename {lib => src}/config/stats.js (100%) rename {lib => src}/irc/BridgedClient.js (100%) rename {lib => src}/irc/ClientPool.js (100%) rename {lib => src}/irc/ConnectionInstance.js (100%) rename {lib => src}/irc/IdentGenerator.js (100%) rename {lib => src}/irc/Ipv6Generator.js (100%) rename {lib => src}/irc/IrcEventBroker.js (100%) rename {lib => src}/irc/IrcServer.js (100%) rename {lib => src}/irc/Scheduler.js (100%) rename {lib => src}/irc/formatting.js (100%) rename {lib => src}/irc/ident.js (100%) rename {lib => src}/logging.js (100%) rename {lib => src}/main.js (100%) rename {lib => src}/models/BridgeRequest.js (100%) rename {lib => src}/models/IrcAction.js (100%) rename {lib => src}/models/IrcClientConfig.js (100%) rename {lib => src}/models/IrcRoom.js (100%) rename {lib => src}/models/IrcUser.js (100%) rename {lib => src}/models/MatrixAction.js (100%) rename {lib => src}/promiseutil.js (100%) rename {lib => src}/provisioning/ProvisionRequest.js (100%) rename {lib => src}/provisioning/Provisioner.js (100%) rename {lib => src}/util/Queue.js (100%) rename {lib => src}/util/QueuePool.js (100%) diff --git a/lib/DataStore.js b/src/DataStore.js similarity index 100% rename from lib/DataStore.js rename to src/DataStore.js diff --git a/lib/DebugApi.js b/src/DebugApi.js similarity index 100% rename from lib/DebugApi.js rename to src/DebugApi.js diff --git a/lib/bridge/IrcBridge.js b/src/bridge/IrcBridge.js similarity index 100% rename from lib/bridge/IrcBridge.js rename to src/bridge/IrcBridge.js diff --git a/lib/bridge/IrcHandler.js b/src/bridge/IrcHandler.js similarity index 100% rename from lib/bridge/IrcHandler.js rename to src/bridge/IrcHandler.js diff --git a/lib/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js similarity index 100% rename from lib/bridge/MatrixHandler.js rename to src/bridge/MatrixHandler.js diff --git a/lib/bridge/MemberListSyncer.js b/src/bridge/MemberListSyncer.js similarity index 100% rename from lib/bridge/MemberListSyncer.js rename to src/bridge/MemberListSyncer.js diff --git a/lib/bridge/PublicitySyncer.js b/src/bridge/PublicitySyncer.js similarity index 100% rename from lib/bridge/PublicitySyncer.js rename to src/bridge/PublicitySyncer.js diff --git a/lib/bridge/QuitDebouncer.js b/src/bridge/QuitDebouncer.js similarity index 100% rename from lib/bridge/QuitDebouncer.js rename to src/bridge/QuitDebouncer.js diff --git a/lib/bridge/RoomAccessSyncer.js b/src/bridge/RoomAccessSyncer.js similarity index 100% rename from lib/bridge/RoomAccessSyncer.js rename to src/bridge/RoomAccessSyncer.js diff --git a/lib/config/schema.yml b/src/config/schema.yml similarity index 100% rename from lib/config/schema.yml rename to src/config/schema.yml diff --git a/lib/config/stats.js b/src/config/stats.js similarity index 100% rename from lib/config/stats.js rename to src/config/stats.js diff --git a/lib/irc/BridgedClient.js b/src/irc/BridgedClient.js similarity index 100% rename from lib/irc/BridgedClient.js rename to src/irc/BridgedClient.js diff --git a/lib/irc/ClientPool.js b/src/irc/ClientPool.js similarity index 100% rename from lib/irc/ClientPool.js rename to src/irc/ClientPool.js diff --git a/lib/irc/ConnectionInstance.js b/src/irc/ConnectionInstance.js similarity index 100% rename from lib/irc/ConnectionInstance.js rename to src/irc/ConnectionInstance.js diff --git a/lib/irc/IdentGenerator.js b/src/irc/IdentGenerator.js similarity index 100% rename from lib/irc/IdentGenerator.js rename to src/irc/IdentGenerator.js diff --git a/lib/irc/Ipv6Generator.js b/src/irc/Ipv6Generator.js similarity index 100% rename from lib/irc/Ipv6Generator.js rename to src/irc/Ipv6Generator.js diff --git a/lib/irc/IrcEventBroker.js b/src/irc/IrcEventBroker.js similarity index 100% rename from lib/irc/IrcEventBroker.js rename to src/irc/IrcEventBroker.js diff --git a/lib/irc/IrcServer.js b/src/irc/IrcServer.js similarity index 100% rename from lib/irc/IrcServer.js rename to src/irc/IrcServer.js diff --git a/lib/irc/Scheduler.js b/src/irc/Scheduler.js similarity index 100% rename from lib/irc/Scheduler.js rename to src/irc/Scheduler.js diff --git a/lib/irc/formatting.js b/src/irc/formatting.js similarity index 100% rename from lib/irc/formatting.js rename to src/irc/formatting.js diff --git a/lib/irc/ident.js b/src/irc/ident.js similarity index 100% rename from lib/irc/ident.js rename to src/irc/ident.js diff --git a/lib/logging.js b/src/logging.js similarity index 100% rename from lib/logging.js rename to src/logging.js diff --git a/lib/main.js b/src/main.js similarity index 100% rename from lib/main.js rename to src/main.js diff --git a/lib/models/BridgeRequest.js b/src/models/BridgeRequest.js similarity index 100% rename from lib/models/BridgeRequest.js rename to src/models/BridgeRequest.js diff --git a/lib/models/IrcAction.js b/src/models/IrcAction.js similarity index 100% rename from lib/models/IrcAction.js rename to src/models/IrcAction.js diff --git a/lib/models/IrcClientConfig.js b/src/models/IrcClientConfig.js similarity index 100% rename from lib/models/IrcClientConfig.js rename to src/models/IrcClientConfig.js diff --git a/lib/models/IrcRoom.js b/src/models/IrcRoom.js similarity index 100% rename from lib/models/IrcRoom.js rename to src/models/IrcRoom.js diff --git a/lib/models/IrcUser.js b/src/models/IrcUser.js similarity index 100% rename from lib/models/IrcUser.js rename to src/models/IrcUser.js diff --git a/lib/models/MatrixAction.js b/src/models/MatrixAction.js similarity index 100% rename from lib/models/MatrixAction.js rename to src/models/MatrixAction.js diff --git a/lib/promiseutil.js b/src/promiseutil.js similarity index 100% rename from lib/promiseutil.js rename to src/promiseutil.js diff --git a/lib/provisioning/ProvisionRequest.js b/src/provisioning/ProvisionRequest.js similarity index 100% rename from lib/provisioning/ProvisionRequest.js rename to src/provisioning/ProvisionRequest.js diff --git a/lib/provisioning/Provisioner.js b/src/provisioning/Provisioner.js similarity index 100% rename from lib/provisioning/Provisioner.js rename to src/provisioning/Provisioner.js diff --git a/lib/util/Queue.js b/src/util/Queue.js similarity index 100% rename from lib/util/Queue.js rename to src/util/Queue.js diff --git a/lib/util/QueuePool.js b/src/util/QueuePool.js similarity index 100% rename from lib/util/QueuePool.js rename to src/util/QueuePool.js From 3a843ceaf8b28bd71344cfece57a71d93f89f304 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 12:57:18 +0100 Subject: [PATCH 002/350] Add typescript support --- .gitignore | 5 +++++ package-lock.json | 6 +++--- package.json | 3 ++- tsconfig.json | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore index f79e38665..c84a56850 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,8 @@ node_modules #passwords passkey.pem + +# Typescript files +*.tsbuildinfo +*.map +lib/ diff --git a/package-lock.json b/package-lock.json index 1d790a7c8..120079790 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3671,9 +3671,9 @@ } }, "typescript": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.3.tgz", - "integrity": "sha512-ACzBtm/PhXBDId6a6sDJfroT2pOWt/oOnk4/dElG5G33ZL776N3Y6/6bKZJBFpd+b05F3Ct9qDjMeJmRWtE2/g==" + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.3.tgz", + "integrity": "sha512-N7bceJL1CtRQ2RiG0AQME13ksR7DiuQh/QehubYcghzv20tnh+MQnQIuJddTmsbqYj+dztchykemz0zFzlvdQw==" }, "uglify-js": { "version": "3.6.0", diff --git a/package.json b/package.json index 805008d78..9494a67a8 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", - "proxyquire": "^1.4.0" + "proxyquire": "^1.4.0", + "typescript": "^3.6.3" } } diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..4c680298f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "incremental": true, + "target": "ES2017", + "module": "commonjs", + "allowJs": true, + "checkJs": false, + "declaration": false, + "sourceMap": true, + "outDir": "./lib", + "composite": false, + "strict": true, + "esModuleInterop": true + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "spec/**/*", + "scripts/**/*", + "app.js" + ] +} From bc8e674231cd953dffaee68f449794bc896cc1fc Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 12:58:08 +0100 Subject: [PATCH 003/350] Add support for TS in pipeline.yml --- .buildkite/pipeline.yml | 3 +++ package.json | 1 + 2 files changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0a703170f..1923243cc 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -10,6 +10,7 @@ steps: - label: ":jasmine: Tests Node 10" command: - "npm install" + - "npm run build" - "npm run test" plugins: - docker#v3.0.1: @@ -18,6 +19,7 @@ steps: - label: ":jasmine: Tests Node 12" command: - "npm install" + - "npm run build" - "npm run test" plugins: - docker#v3.0.1: @@ -26,6 +28,7 @@ steps: - label: ":nyc: Coverage" command: - "npm install" + - "npm run build" - "npm run ci-test" plugins: - docker#v3.0.1: diff --git a/package.json b/package.json index 9494a67a8..12c7711df 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "node": ">=6.9" }, "scripts": { + "build": "npm run build", "test": "BLUEBIRD_DEBUG=1 node --max_old_space_size=3072 node_modules/jasmine/bin/jasmine.js --stop-on-failure=true", "lint": "eslint --max-warnings 0 lib spec", "check": "npm test && npm run lint", From d98e3e4ce621cecd313412c8f08220f3f7a23dc4 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 13:06:46 +0100 Subject: [PATCH 004/350] eslint should lint the src directory --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 12c7711df..f38cb4457 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,9 @@ "node": ">=6.9" }, "scripts": { - "build": "npm run build", + "build": "tsc --project ./tsconfig.json", "test": "BLUEBIRD_DEBUG=1 node --max_old_space_size=3072 node_modules/jasmine/bin/jasmine.js --stop-on-failure=true", - "lint": "eslint --max-warnings 0 lib spec", + "lint": "eslint --max-warnings 0 src spec", "check": "npm test && npm run lint", "ci-test": "node --max_old_space_size=3072 node_modules/nyc/bin/nyc.js --report text jasmine", "ci": "npm run lint && npm run ci-test" From a006c3f9dfe6e9d0ee2e04f5dff36f9a6e4bd13d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 13:08:51 +0100 Subject: [PATCH 005/350] Fix eslint command --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f38cb4457..b39fbcd40 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "tsc --project ./tsconfig.json", "test": "BLUEBIRD_DEBUG=1 node --max_old_space_size=3072 node_modules/jasmine/bin/jasmine.js --stop-on-failure=true", - "lint": "eslint --max-warnings 0 src spec", + "lint": "eslint --max-warnings 0 src/**/*.js app.js spec", "check": "npm test && npm run lint", "ci-test": "node --max_old_space_size=3072 node_modules/nyc/bin/nyc.js --report text jasmine", "ci": "npm run lint && npm run ci-test" From fca2a1f3feabdfdf8c763b53540ae34aad272a06 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 14:47:05 +0100 Subject: [PATCH 006/350] Convert DataStore to typescript --- package-lock.json | 5 + package.json | 1 + src/DataStore.js | 619 --------------------------------------- src/DataStore.ts | 630 ++++++++++++++++++++++++++++++++++++++++ src/DebugApi.js | 2 +- src/bridge/IrcBridge.js | 2 +- 6 files changed, 638 insertions(+), 621 deletions(-) delete mode 100644 src/DataStore.js create mode 100644 src/DataStore.ts diff --git a/package-lock.json b/package-lock.json index 120079790..62ab9de8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -136,6 +136,11 @@ "to-fast-properties": "^2.0.0" } }, + "@types/bluebird": { + "version": "3.5.27", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.27.tgz", + "integrity": "sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==" + }, "@types/chai": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.7.tgz", diff --git a/package.json b/package.json index b39fbcd40..880b61f20 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "winston-daily-rotate-file": "^3.2.1" }, "devDependencies": { + "@types/bluebird": "^3.5.27", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", diff --git a/src/DataStore.js b/src/DataStore.js deleted file mode 100644 index 82efd859e..000000000 --- a/src/DataStore.js +++ /dev/null @@ -1,619 +0,0 @@ -/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping -"use strict"; - -const Promise = require("bluebird"); -const crypto = require('crypto'); - -const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; -const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const RemoteUser = require("matrix-appservice-bridge").RemoteUser; -const IrcRoom = require("./models/IrcRoom"); -const IrcClientConfig = require("./models/IrcClientConfig"); -const log = require("./logging").get("DataStore"); -const fs = require('fs'); - -function DataStore(userStore, roomStore, pkeyPath, bridgeDomain) { - this._roomStore = roomStore; - this._userStore = userStore; - this._serverMappings = {}; // { domain: IrcServer } - this._bridgeDomain = bridgeDomain; - - var errLog = function(fieldName) { - return function(err) { - if (err) { - log.error("Failed to ensure '%s' index on store: " + err, fieldName); - return; - } - log.info("Indexes checked on '%s' for store.", fieldName); - }; - }; - - // add some indexes - this._roomStore.db.ensureIndex({ - fieldName: "id", - unique: true, - sparse: false - }, errLog("id")); - this._roomStore.db.ensureIndex({ - fieldName: "matrix_id", - unique: false, - sparse: true - }, errLog("matrix_id")); - this._roomStore.db.ensureIndex({ - fieldName: "remote_id", - unique: false, - sparse: true - }, errLog("remote_id")); - this._userStore.db.ensureIndex({ - fieldName: "data.localpart", - unique: false, - sparse: true - }, errLog("localpart")); - this._userStore.db.ensureIndex({ - fieldName: "id", - unique: true, - sparse: false - }, errLog("user id")); - - this._privateKey = null; - - if (pkeyPath) { - try { - this._privateKey = fs.readFileSync(pkeyPath, "utf8").toString(); - - // Test whether key is a valid PEM key (publicEncrypt does internal validation) - try { - crypto.publicEncrypt( - this._privateKey, - new Buffer("This is a test!") - ); - } - catch (err) { - log.error(`Failed to validate private key: (${err.message})`); - throw err; - } - - log.info(`Private key loaded from ${pkeyPath} - IRC password encryption enabled.`); - } - catch (err) { - log.error(`Could not load private key ${err.message}.`); - throw err; - } - } - - // Cache as many mappings as possible for hot paths like message sending. - - // TODO: cache IRC channel -> [room_id] mapping (only need to remove them in - // removeRoom() which is infrequent) - // TODO: cache room_id -> [#channel] mapping (only need to remove them in - // removeRoom() which is infrequent) - -} - -DataStore.prototype.setServerFromConfig = Promise.coroutine(function*(server, serverConfig) { - this._serverMappings[server.domain] = server; - - var channels = Object.keys(serverConfig.mappings); - for (var i = 0; i < channels.length; i++) { - var channel = channels[i]; - for (var k = 0; k < serverConfig.mappings[channel].length; k++) { - var ircRoom = new IrcRoom(server, channel); - var mxRoom = new MatrixRoom( - serverConfig.mappings[channel][k] - ); - yield this.storeRoom(ircRoom, mxRoom, 'config'); - } - } - - // Some kinds of users may have the same user_id prefix so will cause ident code to hit - // getMatrixUserByUsername hundreds of times which can be slow: - // https://github.com/matrix-org/matrix-appservice-irc/issues/404 - let domainKey = server.domain.replace(/\./g, "_"); - this._userStore.db.ensureIndex({ - fieldName: "data.client_config." + domainKey + ".username", - unique: true, - sparse: true - }, function(err) { - if (err) { - log.error("Failed to ensure ident username index on users database!"); - return; - } - log.info("Indexes checked for ident username for " + server.domain + " on users database"); - }); -}); - -/** - * Persists an IRC <--> Matrix room mapping in the database. - * @param {IrcRoom} ircRoom : The IRC room to store. - * @param {MatrixRoom} matrixRoom : The Matrix room to store. - * @param {string} origin : "config" if this mapping is from the config yaml, - * "provision" if this mapping was provisioned, "alias" if it was created via - * aliasing and "join" if it was created during a join. - * @return {Promise} - */ -DataStore.prototype.storeRoom = function(ircRoom, matrixRoom, origin) { - if (typeof origin !== 'string') { - throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); - } - - log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", - matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel, origin); - - let mappingId = createMappingId(matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel); - return this._roomStore.linkRooms(matrixRoom, ircRoom, { - origin: origin - }, mappingId); -}; - -/** - * Get an IRC <--> Matrix room mapping from the database. - * @param {string} roomId : The Matrix room ID. - * @param {string} ircDomain : The IRC server domain. - * @param {string} ircChannel : The IRC channel. - * @param {string} origin : (Optional) "config" if this mapping was from the config yaml, - * "provision" if this mapping was provisioned, "alias" if it was created via aliasing and - * "join" if it was created during a join. - * @return {Promise} A promise which resolves to a room entry, or null if one is not found. - */ -DataStore.prototype.getRoom = function(roomId, ircDomain, ircChannel, origin) { - if (typeof origin !== 'undefined' && typeof origin !== 'string') { - throw new Error(`If defined, origin must be a string = - "config"|"provision"|"alias"|"join"`); - } - let mappingId = createMappingId(roomId, ircDomain, ircChannel); - - return this._roomStore.getEntryById(mappingId).then( - (entry) => { - if (origin && entry && origin !== entry.data.origin) { - return null; - } - return entry; - }); -}; - -/** - * Get all Matrix <--> IRC room mappings from the database. - * @return {Promise} A promise which resolves to a map: - * $roomId => [{networkId: 'server #channel1', channel: '#channel2'} , ...] - */ -DataStore.prototype.getAllChannelMappings = Promise.coroutine(function*() { - let entries = yield this._roomStore.select( - { - matrix_id: {$exists: true}, - remote_id: {$exists: true}, - 'remote.type': "channel" - } - ); - - let mappings = {}; - - entries.forEach((e) => { - // drop unknown irc networks in the database - if (!this._serverMappings[e.remote.domain]) { - return; - } - if (!mappings[e.matrix_id]) { - mappings[e.matrix_id] = []; - } - mappings[e.matrix_id].push({ - networkId: this._serverMappings[e.remote.domain].getNetworkId(), - channel: e.remote.channel - }); - }); - - return mappings; -}); - -/** - * Get provisioned IRC <--> Matrix room mappings from the database where - * the matrix room ID is roomId. - * @param {string} roomId : The Matrix room ID. - * @return {Promise} A promise which resolves to a list - * of entries. - */ -DataStore.prototype.getProvisionedMappings = function(roomId) { - return this._roomStore.getEntriesByMatrixId(roomId).filter( - (entry) => { - return entry.data && entry.data.origin === 'provision' - }); -}; - -/** - * Remove an IRC <--> Matrix room mapping from the database. - * @param {string} roomId : The Matrix room ID. - * @param {string} ircDomain : The IRC server domain. - * @param {string} ircChannel : The IRC channel. - * @param {string} origin : "config" if this mapping was from the config yaml, - * "provision" if this mapping was provisioned, "alias" if it was created via - * aliasing and "join" if it was created during a join. - * @return {Promise} - */ -DataStore.prototype.removeRoom = function(roomId, ircDomain, ircChannel, origin) { - if (typeof origin !== 'string') { - throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); - } - - return this._roomStore.delete({ - id: createMappingId(roomId, ircDomain, ircChannel), - 'data.origin': origin - }); -}; - -/** - * Retrieve a list of IRC rooms for a given room ID. - * @param {string} roomId : The room ID to get mapped IRC channels. - * @return {Promise>} A promise which resolves to a list of - * rooms. - */ -DataStore.prototype.getIrcChannelsForRoomId = function(roomId) { - return this._roomStore.getLinkedRemoteRooms(roomId).then((remoteRooms) => { - return remoteRooms.filter((remoteRoom) => { - return Boolean(this._serverMappings[remoteRoom.get("domain")]); - }).map((remoteRoom) => { - let server = this._serverMappings[remoteRoom.get("domain")]; - return IrcRoom.fromRemoteRoom(server, remoteRoom); - }); - }); -}; - -/** - * Retrieve a list of IRC rooms for a given list of room IDs. This is significantly - * faster than calling getIrcChannelsForRoomId for each room ID. - * @param {string[]} roomIds : The room IDs to get mapped IRC channels. - * @return {Promise>} A promise which resolves to a map of - * room ID to an array of IRC rooms. - */ -DataStore.prototype.getIrcChannelsForRoomIds = function(roomIds) { - return this._roomStore.batchGetLinkedRemoteRooms(roomIds).then((roomIdToRemoteRooms) => { - Object.keys(roomIdToRemoteRooms).forEach((roomId) => { - // filter out rooms with unknown IRC servers and - // map RemoteRooms to IrcRooms - roomIdToRemoteRooms[roomId] = roomIdToRemoteRooms[roomId].filter((remoteRoom) => { - return Boolean(this._serverMappings[remoteRoom.get("domain")]); - }).map((remoteRoom) => { - let server = this._serverMappings[remoteRoom.get("domain")]; - return IrcRoom.fromRemoteRoom(server, remoteRoom); - }); - }); - return roomIdToRemoteRooms; - }); -}; - -/** - * Retrieve a list of Matrix rooms for a given server and channel. - * @param {IrcServer} server : The server to get rooms for. - * @param {string} channel : The channel to get mapped rooms for. - * @return {Promise>} A promise which resolves to a list of rooms. - */ -DataStore.prototype.getMatrixRoomsForChannel = function(server, channel) { - var ircRoom = new IrcRoom(server, channel); - return this._roomStore.getLinkedMatrixRooms( - IrcRoom.createId(ircRoom.getServer(), ircRoom.getChannel()) - ); -}; - -DataStore.prototype.getMappingsForChannelByOrigin = function(server, channel, origin, allowUnset) { - if (typeof origin === "string") { - origin = [origin]; - } - if (!Array.isArray(origin) || !origin.every((s) => typeof s === "string")) { - throw new Error("origin must be string or array of strings"); - } - let remoteId = IrcRoom.createId(server, channel); - return this._roomStore.getEntriesByRemoteId(remoteId).then((entries) => { - return entries.filter((e) => { - if (allowUnset) { - if (!e.data || !e.data.origin) { - return true; - } - } - return e.data && origin.indexOf(e.data.origin) !== -1; - }); - }); -}; - -DataStore.prototype.getModesForChannel = function (server, channel) { - log.info("getModesForChannel (server=%s, channel=%s)", - server.domain, channel - ); - let remoteId = IrcRoom.createId(server, channel); - return this._roomStore.getEntriesByRemoteId(remoteId).then((entries) => { - const mapping = {}; - entries.forEach((entry) => { - mapping[entry.matrix.getId()] = entry.remote.get("modes") || []; - }); - return mapping; - }); -}; - -DataStore.prototype.setModeForRoom = Promise.coroutine(function*(roomId, mode, enabled=True) { - log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", - mode, roomId, enabled - ); - return this._roomStore.getEntriesByMatrixId(roomId).then((entries) => { - entries.map((entry) => { - const modes = entry.remote.get("modes") || []; - const hasMode = modes.includes(mode); - - if (hasMode === enabled) { - return; - } - if (enabled) { - modes.push(mode); - } - else { - modes.splice(modes.indexOf(mode), 1); - } - - entry.remote.set("modes", modes); - - this._roomStore.upsertEntry(entry); - }); - }); -}); - -DataStore.prototype.setPmRoom = function(ircRoom, matrixRoom, userId, virtualUserId) { - log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)", - matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId, - virtualUserId); - - return this._roomStore.linkRooms(matrixRoom, ircRoom, { - real_user_id: userId, - virtual_user_id: virtualUserId - }, createPmId(userId, virtualUserId)); -}; - -DataStore.prototype.getMatrixPmRoom = function(realUserId, virtualUserId) { - var id = createPmId(realUserId, virtualUserId); - return this._roomStore.getEntryById(id).then(function(entry) { - if (!entry) { - return null; - } - return entry.matrix; - }); -}; - -DataStore.prototype.getTrackedChannelsForServer = function(ircAddr) { - return this._roomStore.getEntriesByRemoteRoomData({ domain: ircAddr }).then( - (entries) => { - var channels = []; - entries.forEach((e) => { - let r = e.remote; - let server = this._serverMappings[r.get("domain")]; - if (!server) { - return; - } - let ircRoom = IrcRoom.fromRemoteRoom(server, r); - if (ircRoom.getType() === "channel") { - channels.push(ircRoom.getChannel()); - } - }); - return channels; - }); -}; - -DataStore.prototype.getRoomIdsFromConfig = function() { - return this._roomStore.getEntriesByLinkData({ - origin: 'config' - }).then(function(entries) { - return entries.map((e) => { - return e.matrix.getId(); - }); - }); -}; - -DataStore.prototype.removeConfigMappings = function() { - return this._roomStore.removeEntriesByLinkData({ - from_config: true // for backwards compatibility - }).then(() => { - return this._roomStore.removeEntriesByLinkData({ - origin: 'config' - }) - }); -}; - -DataStore.prototype.getIpv6Counter = Promise.coroutine(function*() { - let config = yield this._userStore.getRemoteUser("config"); - if (!config) { - config = new RemoteUser("config"); - config.set("ipv6_counter", 0); - yield this._userStore.setRemoteUser(config); - } - return config.get("ipv6_counter"); -}); - -DataStore.prototype.setIpv6Counter = Promise.coroutine(function*(counter) { - let config = yield this._userStore.getRemoteUser("config"); - if (!config) { - config = new RemoteUser("config"); - } - config.set("ipv6_counter", counter); - yield this._userStore.setRemoteUser(config); -}); - -/** - * Retrieve a stored admin room based on the room's ID. - * @param {String} roomId : The room ID of the admin room. - * @return {Promise} Resolved when the room is retrieved. - */ -DataStore.prototype.getAdminRoomById = function(roomId) { - return this._roomStore.getEntriesByMatrixId(roomId).then(function(entries) { - if (entries.length == 0) { - return null; - } - if (entries.length > 1) { - log.error("getAdminRoomById(" + roomId + ") has " + entries.length + " entries"); - } - if (entries[0].matrix.get("admin_id")) { - return entries[0].matrix; - } - return null; - }); -}; - -/** - * Stores a unique admin room for a given user ID. - * @param {MatrixRoom} room : The matrix room which is the admin room for this user. - * @param {String} userId : The user ID who is getting an admin room. - * @return {Promise} Resolved when the room is stored. - */ -DataStore.prototype.storeAdminRoom = function(room, userId) { - log.info("storeAdminRoom (id=%s, user_id=%s)", room.getId(), userId); - room.set("admin_id", userId); - return this._roomStore.upsertEntry({ - id: createAdminId(userId), - matrix: room, - }); -}; - -DataStore.prototype.upsertRoomStoreEntry = function(entry) { - return this._roomStore.upsertEntry(entry); -} - -DataStore.prototype.getAdminRoomByUserId = function(userId) { - return this._roomStore.getEntryById(createAdminId(userId)).then(function(entry) { - if (!entry) { - return null; - } - return entry.matrix; - }); -}; - -DataStore.prototype.storeMatrixUser = function(matrixUser) { - return this._userStore.setMatrixUser(matrixUser); -}; - -DataStore.prototype.getMatrixUserByLocalpart = function(localpart) { - return this._userStore.getMatrixUser(`@${localpart}:${this._bridgeDomain}`); -}; - -DataStore.prototype.getIrcClientConfig = function(userId, domain) { - return this._userStore.getMatrixUser(userId).then((matrixUser) => { - if (!matrixUser) { - return null; - } - var userConfig = matrixUser.get("client_config"); - if (!userConfig) { - return null; - } - // map back from _ to . - Object.keys(userConfig).forEach(function(domainWithUnderscores) { - let actualDomain = domainWithUnderscores.replace(/_/g, "."); - if (actualDomain !== domainWithUnderscores) { // false for 'localhost' - userConfig[actualDomain] = userConfig[domainWithUnderscores]; - delete userConfig[domainWithUnderscores]; - } - }) - var configData = userConfig[domain]; - if (!configData) { - return null; - } - let clientConfig = new IrcClientConfig(userId, domain, configData); - if (clientConfig.getPassword()) { - if (!this._privateKey) { - throw new Error(`Cannot decrypt password of ${userId} - no private key`); - } - let decryptedPass = crypto.privateDecrypt( - this._privateKey, - new Buffer(clientConfig.getPassword(), 'base64') - ).toString(); - // Extract the password by removing the prefixed salt and seperating space - decryptedPass = decryptedPass.split(' ')[1]; - clientConfig.setPassword(decryptedPass); - } - return clientConfig; - }); -}; - -DataStore.prototype.storeIrcClientConfig = function(config) { - return this._userStore.getMatrixUser(config.getUserId()).then((user) => { - if (!user) { - user = new MatrixUser(config.getUserId()); - } - var userConfig = user.get("client_config") || {}; - if (config.getPassword()) { - if (!this._privateKey) { - throw new Error( - 'Cannot store plaintext passwords' - ); - } - let salt = crypto.randomBytes(16).toString('base64'); - let encryptedPass = crypto.publicEncrypt( - this._privateKey, - new Buffer(salt + ' ' + config.getPassword()) - ).toString('base64'); - // Store the encrypted password, ready for the db - config.setPassword(encryptedPass); - } - userConfig[config.getDomain().replace(/\./g, "_")] = config.serialize(); - user.set("client_config", userConfig); - return this._userStore.setMatrixUser(user); - }); -}; - -DataStore.prototype.getUserFeatures = function(userId) { - return this._userStore.getMatrixUser(userId).then((matrixUser) => { - return matrixUser ? (matrixUser.get("features") || {}) : {}; - }); -}; - -DataStore.prototype.storeUserFeatures = function(userId, features) { - return this._userStore.getMatrixUser(userId).then((matrixUser) => { - if (!matrixUser) { - matrixUser = new MatrixUser(userId); - } - matrixUser.set("features", features); - return this._userStore.setMatrixUser(matrixUser); - }); -}; - -DataStore.prototype.storePass = Promise.coroutine( - function*(userId, domain, pass) { - let config = yield this.getIrcClientConfig(userId, domain); - if (!config) { - throw new Error(`${userId} does not have an IRC client configured for ${domain}`); - } - config.setPassword(pass); - yield this.storeIrcClientConfig(config); - } -); - -DataStore.prototype.removePass = Promise.coroutine( - function*(userId, domain) { - let config = yield this.getIrcClientConfig(userId, domain); - config.setPassword(undefined); - yield this.storeIrcClientConfig(config); - } -); - -DataStore.prototype.getMatrixUserByUsername = Promise.coroutine( -function*(domain, username) { - let domainKey = domain.replace(/\./g, "_"); - let matrixUsers = yield this._userStore.getByMatrixData({ - ["client_config." + domainKey + ".username"]: username - }); - - if (matrixUsers.length > 1) { - log.error( - "getMatrixUserByUsername return %s results for %s on %s", - matrixUsers.length, username, domain - ); - } - return matrixUsers[0]; -}); - -function createPmId(userId, virtualUserId) { - // space as delimiter as none of these IDs allow spaces. - return "PM_" + userId + " " + virtualUserId; // clobber based on this. -} - -function createAdminId(userId) { - return "ADMIN_" + userId; // clobber based on this. -} - -function createMappingId(roomId, ircDomain, ircChannel) { - // space as delimiter as none of these IDs allow spaces. - return roomId + " " + ircDomain + " " + ircChannel; // clobber based on this -} - -module.exports = DataStore; diff --git a/src/DataStore.ts b/src/DataStore.ts new file mode 100644 index 000000000..4c5688e26 --- /dev/null +++ b/src/DataStore.ts @@ -0,0 +1,630 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import * as crypto from "crypto"; +import * as fs from "fs"; +import {default as Bluebird} from "bluebird"; + +// Ignore definition errors for now. +//@ts-ignore +import { MatrixRoom, RemoteRoom, MatrixUser, RemoteUser} from "matrix-appservice-bridge"; + +interface RoomEntry { + id: string; + matrix: MatrixRoom; + remote: RemoteRoom; + data: any; +} + +interface ChannelMappings { + [roomId: string]: Array<{networkId: string, channel: string}> +} + +interface UserFeatures { + [name: string]: boolean +} + +const IrcRoom = require("./models/IrcRoom"); +const IrcClientConfig = require("./models/IrcClientConfig"); +const log = require("./logging").get("DataStore"); + +export type RoomOrigin = "config"|"provision"|"alias"|"join"; + +export class DataStore { + private serverMappings: {[domain: string]: any /*IrcServer*/} = {}; + private privateKey: string|null; + constructor( + private userStore: any, + private roomStore: any, + pkeyPath: string, + private bridgeDomain: string) { + const errLog = function(fieldName: string) { + return (err: Error) => { + if (err) { + log.error("Failed to ensure '%s' index on store: " + err, fieldName); + return; + } + log.info("Indexes checked on '%s' for store.", fieldName); + }; + }; + + // add some indexes + this.roomStore.db.ensureIndex({ + fieldName: "id", + unique: true, + sparse: false + }, errLog("id")); + this.roomStore.db.ensureIndex({ + fieldName: "matrix_id", + unique: false, + sparse: true + }, errLog("matrix_id")); + this.roomStore.db.ensureIndex({ + fieldName: "remote_id", + unique: false, + sparse: true + }, errLog("remote_id")); + this.userStore.db.ensureIndex({ + fieldName: "data.localpart", + unique: false, + sparse: true + }, errLog("localpart")); + this.userStore.db.ensureIndex({ + fieldName: "id", + unique: true, + sparse: false + }, errLog("user id")); + + this.privateKey = null; + + if (pkeyPath) { + try { + this.privateKey = fs.readFileSync(pkeyPath, "utf8").toString(); + + // Test whether key is a valid PEM key (publicEncrypt does internal validation) + try { + crypto.publicEncrypt( + this.privateKey, + new Buffer("This is a test!") + ); + } + catch (err) { + log.error(`Failed to validate private key: (${err.message})`); + throw err; + } + + log.info(`Private key loaded from ${pkeyPath} - IRC password encryption enabled.`); + } + catch (err) { + log.error(`Could not load private key ${err.message}.`); + throw err; + } + } + // Cache as many mappings as possible for hot paths like message sending. + + // TODO: cache IRC channel -> [room_id] mapping (only need to remove them in + // removeRoom() which is infrequent) + // TODO: cache room_id -> [#channel] mapping (only need to remove them in + // removeRoom() which is infrequent) + } + + public async setServerFromConfig(server: any, serverConfig: any): Promise { + this.serverMappings[server.domain] = server; + + for (const channel of Object.keys(serverConfig.mappings)) { + const ircRoom = new IrcRoom(server, channel); + for (const roomId of serverConfig.mappings[channel]) { + const mxRoom = new MatrixRoom(roomId); + await this.storeRoom(ircRoom, mxRoom, "config"); + } + } + + // Some kinds of users may have the same user_id prefix so will cause ident code to hit + // getMatrixUserByUsername hundreds of times which can be slow: + // https://github.com/matrix-org/matrix-appservice-irc/issues/404 + const domainKey = server.domain.replace(/\./g, "_"); + this.userStore.db.ensureIndex({ + fieldName: "data.client_config." + domainKey + ".username", + unique: true, + sparse: true + }, (err: Error) => { + if (err) { + log.error("Failed to ensure ident username index on users database!"); + return; + } + log.info("Indexes checked for ident username for " + server.domain + " on users database"); + }); + } + + /** + * Persists an IRC <--> Matrix room mapping in the database. + * @param {IrcRoom} ircRoom : The IRC room to store. + * @param {MatrixRoom} matrixRoom : The Matrix room to store. + * @param {string} origin : "config" if this mapping is from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via + * aliasing and "join" if it was created during a join. + * @return {Promise} + */ + public async storeRoom(ircRoom: any, matrixRoom: any, origin: RoomOrigin): Promise { + if (typeof origin !== "string") { + throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); + } + + log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", + matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel, origin); + + const mappingId = DataStore.createMappingId(matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel); + await this.roomStore.linkRooms(matrixRoom, ircRoom, { + origin: origin + }, mappingId); + }; + + /** + * Get an IRC <--> Matrix room mapping from the database. + * @param {string} roomId : The Matrix room ID. + * @param {string} ircDomain : The IRC server domain. + * @param {string} ircChannel : The IRC channel. + * @param {string} origin : (Optional) "config" if this mapping was from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via aliasing and + * "join" if it was created during a join. + * @return {Promise} A promise which resolves to a room entry, or null if one is not found. + */ + public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise { + if (typeof origin !== "undefined" && typeof origin !== "string") { + throw new Error(`If defined, origin must be a string = + "config"|"provision"|"alias"|"join"`); + } + const mappingId = DataStore.createMappingId(roomId, ircDomain, ircChannel); + return this.roomStore.getEntryById(mappingId).then( + (entry: RoomEntry) => { + if (origin && entry && origin !== entry.data.origin) { + return null; + } + return entry; + }); + } + + /** + * Get all Matrix <--> IRC room mappings from the database. + * @return {Promise} A promise which resolves to a map: + * $roomId => [{networkId: 'server #channel1', channel: '#channel2'} , ...] + */ + public async getAllChannelMappings(): Promise { + const entries = await this.roomStore.select( + { + matrix_id: {$exists: true}, + remote_id: {$exists: true}, + 'remote.type': "channel" + } + ); + + const mappings: ChannelMappings = {}; + + entries.forEach((e: any) => { + // drop unknown irc networks in the database + if (!this.serverMappings[e.remote.domain]) { + return; + } + if (!mappings[e.matrix_id]) { + mappings[e.matrix_id] = new Array(); + } + mappings[e.matrix_id].push({ + networkId: this.serverMappings[e.remote.domain].getNetworkId(), + channel: e.remote.channel + }); + }); + + return mappings; + } + + /** + * Get provisioned IRC <--> Matrix room mappings from the database where + * the matrix room ID is roomId. + * @param {string} roomId : The Matrix room ID. + * @return {Promise} A promise which resolves to a list + * of entries. + */ + public getProvisionedMappings(roomId: string): Bluebird { + return Bluebird.cast(this.roomStore.getEntriesByMatrixId(roomId)).filter( + (entry: RoomEntry) => { + return entry.data && entry.data.origin === 'provision' + } + ); + } + + /** + * Remove an IRC <--> Matrix room mapping from the database. + * @param {string} roomId : The Matrix room ID. + * @param {string} ircDomain : The IRC server domain. + * @param {string} ircChannel : The IRC channel. + * @param {string} origin : "config" if this mapping was from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via + * aliasing and "join" if it was created during a join. + * @return {Promise} + */ + public async removeRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise { + if (typeof origin !== 'string') { + throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); + } + + return await this.roomStore.delete({ + id: DataStore.createMappingId(roomId, ircDomain, ircChannel), + 'data.origin': origin + }); + } + + /** + * Retrieve a list of IRC rooms for a given room ID. + * @param {string} roomId : The room ID to get mapped IRC channels. + * @return {Promise>} A promise which resolves to a list of + * rooms. + */ + public async getIrcChannelsForRoomId(roomId: string): Promise { + return this.roomStore.getLinkedRemoteRooms(roomId).then((remoteRooms: RemoteRoom[]) => { + return remoteRooms.filter((remoteRoom) => { + return Boolean(this.serverMappings[remoteRoom.get("domain")]); + }).map((remoteRoom) => { + let server = this.serverMappings[remoteRoom.get("domain")]; + return IrcRoom.fromRemoteRoom(server, remoteRoom); + }); + }); + } + + /** + * Retrieve a list of IRC rooms for a given list of room IDs. This is significantly + * faster than calling getIrcChannelsForRoomId for each room ID. + * @param {string[]} roomIds : The room IDs to get mapped IRC channels. + * @return {Promise>} A promise which resolves to a map of + * room ID to an array of IRC rooms. + */ + public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: any[] /*IrcRoom*/ }> { + const roomIdToRemoteRooms: {[roomId: string]: RemoteRoom[]} = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); + for (const roomId of Object.keys(roomIdToRemoteRooms)) { + // filter out rooms with unknown IRC servers and + // map RemoteRooms to IrcRooms + roomIdToRemoteRooms[roomId] = roomIdToRemoteRooms[roomId].filter((remoteRoom) => { + return Boolean(this.serverMappings[remoteRoom.get("domain")]); + }).map((remoteRoom) => { + const server = this.serverMappings[remoteRoom.get("domain")]; + return IrcRoom.fromRemoteRoom(server, remoteRoom); + }); + } + return roomIdToRemoteRooms; + } + + /** + * Retrieve a list of Matrix rooms for a given server and channel. + * @param {IrcServer} server : The server to get rooms for. + * @param {string} channel : The channel to get mapped rooms for. + * @return {Promise>} A promise which resolves to a list of rooms. + */ + public async getMatrixRoomsForChannel(server: any, channel: string): Promise> { + const ircRoom = new IrcRoom(server, channel); + return await this.roomStore.getLinkedMatrixRooms( + IrcRoom.createId(ircRoom.getServer(), ircRoom.getChannel()) + ); + } + + public async getMappingsForChannelByOrigin(server: any, channel: string, origin: RoomOrigin|RoomOrigin[], allowUnset: boolean) { + if (typeof origin === "string") { + origin = [origin]; + } + + if (!Array.isArray(origin) || !origin.every((s) => typeof s === "string")) { + throw new Error("origin must be string or array of strings"); + } + + const remoteId = IrcRoom.createId(server, channel); + return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: RoomEntry[]) => { + return entries.filter((e) => { + if (allowUnset) { + if (!e.data || !e.data.origin) { + return true; + } + } + return e.data && origin.indexOf(e.data.origin) !== -1; + }); + }); + } + + public async getModesForChannel (server: any, channel: string): Promise<{[id: string]: string}> { + log.info("getModesForChannel (server=%s, channel=%s)", + server.domain, channel + ); + const remoteId = IrcRoom.createId(server, channel); + return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: RoomEntry[]) => { + const mapping: {[id: string]: string} = {}; + entries.forEach((entry) => { + mapping[entry.matrix.getId()] = entry.remote.get("modes") || []; + }); + return mapping; + }); + } + + public async setModeForRoom(roomId: string, mode: string, enabled: boolean = true): Promise { + log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", + mode, roomId, enabled + ); + const entries: RoomEntry[] = await this.roomStore.getEntriesByMatrixId(roomId); + for (const entry of entries) { + const modes = entry.remote.get("modes") || []; + const hasMode = modes.includes(mode); + + if (hasMode === enabled) { + continue; + } + if (enabled) { + modes.push(mode); + } + else { + modes.splice(modes.indexOf(mode), 1); + } + + entry.remote.set("modes", modes); + + this.roomStore.upsertEntry(entry); + } + } + + public async setPmRoom(ircRoom: any, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise { + log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)", + matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId, + virtualUserId); + + await this.roomStore.linkRooms(matrixRoom, ircRoom, { + real_user_id: userId, + virtual_user_id: virtualUserId + }, DataStore.createPmId(userId, virtualUserId)); + } + + public async getMatrixPmRoom(realUserId: string, virtualUserId: string) { + const id = DataStore.createPmId(realUserId, virtualUserId); + const entry = await this.roomStore.getEntryById(id); + if (!entry) { + return null; + } + return entry.matrix; + } + + public async getTrackedChannelsForServer(domain: string) { + const entries: RoomEntry[] = await this.roomStore.getEntriesByRemoteRoomData({ domain }); + const channels: string[] = []; + entries.forEach((e) => { + const r = e.remote; + const server = this.serverMappings[r.get("domain")]; + if (!server) { + return; + } + const ircRoom = IrcRoom.fromRemoteRoom(server, r); + if (ircRoom.getType() === "channel") { + channels.push(ircRoom.getChannel()); + } + }); + return channels; + } + + public async getRoomIdsFromConfig() { + const entries: RoomEntry[] = await this.roomStore.getEntriesByLinkData({ + origin: 'config' + }); + return entries.map((e) => { + return e.matrix.getId(); + }); + } + + public async removeConfigMappings() { + await this.roomStore.removeEntriesByLinkData({ + from_config: true // for backwards compatibility + }); + await this.roomStore.removeEntriesByLinkData({ + origin: 'config' + }); + } + + public async getIpv6Counter(): Promise { + let config = await this.userStore.getRemoteUser("config"); + if (!config) { + config = new RemoteUser("config"); + config.set("ipv6_counter", 0); + await this.userStore.setRemoteUser(config); + } + return config.get("ipv6_counter"); + } + + + public async setIpv6Counter(counter: number) { + let config = await this.userStore.getRemoteUser("config"); + if (!config) { + config = new RemoteUser("config"); + } + config.set("ipv6_counter", counter); + await this.userStore.setRemoteUser(config); + } + + /** + * Retrieve a stored admin room based on the room's ID. + * @param {String} roomId : The room ID of the admin room. + * @return {Promise} Resolved when the room is retrieved. + */ + public async getAdminRoomById(roomId: string): Promise { + const entries: RoomEntry[] = await this.roomStore.getEntriesByMatrixId(roomId); + if (entries.length == 0) { + return null; + } + if (entries.length > 1) { + log.error("getAdminRoomById(" + roomId + ") has " + entries.length + " entries"); + } + if (entries[0].matrix.get("admin_id")) { + return entries[0].matrix; + } + return null; + } + + /** + * Stores a unique admin room for a given user ID. + * @param {MatrixRoom} room : The matrix room which is the admin room for this user. + * @param {String} userId : The user ID who is getting an admin room. + * @return {Promise} Resolved when the room is stored. + */ + public async storeAdminRoom(room: MatrixRoom, userId: string): Promise { + log.info("storeAdminRoom (id=%s, user_id=%s)", room.getId(), userId); + room.set("admin_id", userId); + await this.roomStore.upsertEntry({ + id: DataStore.createAdminId(userId), + matrix: room, + }); + } + + public async upsertRoomStoreEntry(entry: RoomEntry): Promise { + await this.roomStore.upsertEntry(entry); + } + + public async getAdminRoomByUserId(userId: string): Promise { + const entry = await this.roomStore.getEntryById(DataStore.createAdminId(userId)); + if (!entry) { + return null; + } + return entry.matrix; + } + + public async storeMatrixUser(matrixUser: MatrixUser): Promise { + await this.userStore.setMatrixUser(matrixUser); + } + + public async getIrcClientConfig(userId: string, domain: string): Promise /*IrcClientConfig*/ { + const matrixUser = await this.userStore.getMatrixUser(userId); + if (!matrixUser) { + return null; + } + const userConfig = matrixUser.get("client_config"); + if (!userConfig) { + return null; + } + // map back from _ to . + Object.keys(userConfig).forEach(function(domainWithUnderscores) { + let actualDomain = domainWithUnderscores.replace(/_/g, "."); + if (actualDomain !== domainWithUnderscores) { // false for 'localhost' + userConfig[actualDomain] = userConfig[domainWithUnderscores]; + delete userConfig[domainWithUnderscores]; + } + }) + const configData = userConfig[domain]; + if (!configData) { + return null; + } + const clientConfig = new IrcClientConfig(userId, domain, configData); + if (clientConfig.getPassword()) { + if (!this.privateKey) { + throw new Error(`Cannot decrypt password of ${userId} - no private key`); + } + let decryptedPass = crypto.privateDecrypt( + this.privateKey, + new Buffer(clientConfig.getPassword(), 'base64') + ).toString(); + // Extract the password by removing the prefixed salt and seperating space + decryptedPass = decryptedPass.split(' ')[1]; + clientConfig.setPassword(decryptedPass); + } + return clientConfig; + } + + public async getMatrixUserByLocalpart(localpart: string): Promise { + return await this.userStore.getMatrixUser(`@${localpart}:${this.bridgeDomain}`); + } + + public async storeIrcClientConfig(config: any /*IrcConfig*/) { + let user = await this.userStore.getMatrixUser(config.getUserId()); + if (!user) { + user = new MatrixUser(config.getUserId()); + } + const userConfig = user.get("client_config") || {}; + if (config.getPassword()) { + if (!this.privateKey) { + throw new Error( + 'Cannot store plaintext passwords' + ); + } + const salt = crypto.randomBytes(16).toString('base64'); + const encryptedPass = crypto.publicEncrypt( + this.privateKey, + new Buffer(salt + ' ' + config.getPassword()) + ).toString('base64'); + // Store the encrypted password, ready for the db + config.setPassword(encryptedPass); + } + userConfig[config.getDomain().replace(/\./g, "_")] = config.serialize(); + user.set("client_config", userConfig); + await this.userStore.setMatrixUser(user); + } + + public async getUserFeatures(userId: string): Promise { + const matrixUser = await this.userStore.getMatrixUser(userId); + return matrixUser ? (matrixUser.get("features") || {}) : {}; + } + + public async storeUserFeatures(userId: string, features: UserFeatures) { + let matrixUser = await this.userStore.getMatrixUser(userId); + if (!matrixUser) { + matrixUser = new MatrixUser(userId); + } + matrixUser.set("features", features); + await this.userStore.setMatrixUser(matrixUser); + } + + public async storePass(userId: string, domain: string, pass: string) { + let config = await this.getIrcClientConfig(userId, domain); + if (!config) { + throw new Error(`${userId} does not have an IRC client configured for ${domain}`); + } + config.setPassword(pass); + await this.storeIrcClientConfig(config); + } + + public async removePass(userId: string, domain: string) { + const config = await this.getIrcClientConfig(userId, domain); + config.setPassword(undefined); + await this.storeIrcClientConfig(config); + } + + public async getMatrixUserByUsername(domain: string, username: string) { + const domainKey = domain.replace(/\./g, "_"); + const matrixUsers = await this.userStore.getByMatrixData({ + ["client_config." + domainKey + ".username"]: username + }); + + if (matrixUsers.length > 1) { + log.error( + "getMatrixUserByUsername return %s results for %s on %s", + matrixUsers.length, username, domain + ); + } + return matrixUsers[0]; + } + + private static createPmId(userId: string, virtualUserId: string) { + // space as delimiter as none of these IDs allow spaces. + return "PM_" + userId + " " + virtualUserId; // clobber based on this. + } + + private static createAdminId(userId: string) { + return "ADMIN_" + userId; // clobber based on this. + } + + private static createMappingId(roomId: string, ircDomain: string, ircChannel: string) { + // space as delimiter as none of these IDs allow spaces. + return roomId + " " + ircDomain + " " + ircChannel; // clobber based on this + } +} \ No newline at end of file diff --git a/src/DebugApi.js b/src/DebugApi.js index 1ec455770..0942c630c 100644 --- a/src/DebugApi.js +++ b/src/DebugApi.js @@ -292,7 +292,7 @@ Remove Alias: ${remove_alias}`); } // Drop room from room store. - this.ircBridge.getStore().removeRoom( + yield this.ircBridge.getStore().removeRoom( roomId, domain, channel, diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index bc2e00333..e37027065 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -18,7 +18,7 @@ var IrcRoom = require("../models/IrcRoom"); var IrcClientConfig = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); -var DataStore = require("../DataStore"); +const { DataStore } = require("../DataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, From 002f1c8a58e70f03819a892a5f073c7ce6ef0d1d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 15:14:44 +0100 Subject: [PATCH 007/350] Convert models/IrcRoom to Typescript --- src/DataStore.ts | 14 +++---- src/bridge/IrcBridge.js | 2 +- src/bridge/IrcHandler.js | 2 +- src/bridge/MatrixHandler.js | 2 +- src/irc/BridgedClient.js | 2 +- src/models/IrcRoom.js | 54 -------------------------- src/models/IrcRoom.ts | 67 +++++++++++++++++++++++++++++++++ src/provisioning/Provisioner.js | 2 +- 8 files changed, 79 insertions(+), 66 deletions(-) delete mode 100644 src/models/IrcRoom.js create mode 100644 src/models/IrcRoom.ts diff --git a/src/DataStore.ts b/src/DataStore.ts index 4c5688e26..baf67bbf0 100644 --- a/src/DataStore.ts +++ b/src/DataStore.ts @@ -17,6 +17,7 @@ limitations under the License. import * as crypto from "crypto"; import * as fs from "fs"; import {default as Bluebird} from "bluebird"; +import { IrcRoom } from "./models/IrcRoom"; // Ignore definition errors for now. //@ts-ignore @@ -37,7 +38,6 @@ interface UserFeatures { [name: string]: boolean } -const IrcRoom = require("./models/IrcRoom"); const IrcClientConfig = require("./models/IrcClientConfig"); const log = require("./logging").get("DataStore"); @@ -158,15 +158,15 @@ export class DataStore { * aliasing and "join" if it was created during a join. * @return {Promise} */ - public async storeRoom(ircRoom: any, matrixRoom: any, origin: RoomOrigin): Promise { + public async storeRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, origin: RoomOrigin): Promise { if (typeof origin !== "string") { throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); } log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", - matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel, origin); + matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin); - const mappingId = DataStore.createMappingId(matrixRoom.getId(), ircRoom.get("domain"), ircRoom.channel); + const mappingId = DataStore.createMappingId(matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel); await this.roomStore.linkRooms(matrixRoom, ircRoom, { origin: origin }, mappingId); @@ -272,7 +272,7 @@ export class DataStore { * @return {Promise>} A promise which resolves to a list of * rooms. */ - public async getIrcChannelsForRoomId(roomId: string): Promise { + public async getIrcChannelsForRoomId(roomId: string): Promise { return this.roomStore.getLinkedRemoteRooms(roomId).then((remoteRooms: RemoteRoom[]) => { return remoteRooms.filter((remoteRoom) => { return Boolean(this.serverMappings[remoteRoom.get("domain")]); @@ -290,7 +290,7 @@ export class DataStore { * @return {Promise>} A promise which resolves to a map of * room ID to an array of IRC rooms. */ - public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: any[] /*IrcRoom*/ }> { + public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: IrcRoom[]}> { const roomIdToRemoteRooms: {[roomId: string]: RemoteRoom[]} = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); for (const roomId of Object.keys(roomIdToRemoteRooms)) { // filter out rooms with unknown IRC servers and @@ -379,7 +379,7 @@ export class DataStore { } } - public async setPmRoom(ircRoom: any, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise { + public async setPmRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise { log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)", matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId, virtualUserId); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index e37027065..17707ba81 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -14,7 +14,7 @@ var ClientPool = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); var IrcUser = require("../models/IrcUser"); -var IrcRoom = require("../models/IrcRoom"); +const { IrcRoom } = require("../models/IrcRoom"); var IrcClientConfig = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index 4589bd785..a3e7b3fc0 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -3,7 +3,7 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); const BridgeRequest = require("../models/BridgeRequest"); -const IrcRoom = require("../models/IrcRoom"); +const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const MatrixAction = require("../models/MatrixAction"); diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index 6ca45022f..a420f384a 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -4,7 +4,7 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; -const IrcRoom = require("../models/IrcRoom"); +const { IrcRoom } = require("../models/IrcRoom"); const MatrixAction = require("../models/MatrixAction"); const IrcAction = require("../models/IrcAction"); const IrcClientConfig = require("../models/IrcClientConfig"); diff --git a/src/irc/BridgedClient.js b/src/irc/BridgedClient.js index 5150717e6..3dafe6a04 100644 --- a/src/irc/BridgedClient.js +++ b/src/irc/BridgedClient.js @@ -7,7 +7,7 @@ const util = require("util"); const EventEmitter = require("events").EventEmitter; const ident = require("./ident"); const ConnectionInstance = require("./ConnectionInstance"); -const IrcRoom = require("../models/IrcRoom"); +const { IrcRoom } = require("../models/IrcRoom"); const log = require("../logging").get("BridgedClient"); // The length of time to wait before trying to join the channel again diff --git a/src/models/IrcRoom.js b/src/models/IrcRoom.js deleted file mode 100644 index 0e4cf1cfc..000000000 --- a/src/models/IrcRoom.js +++ /dev/null @@ -1,54 +0,0 @@ -"use strict"; -const RemoteRoom = require("matrix-appservice-bridge").RemoteRoom; -const toIrcLowerCase = require("../irc/formatting").toIrcLowerCase; - -class IrcRoom extends RemoteRoom { - /** - * Construct a new IRC room. - * @constructor - * @param {IrcServer} server : The IRC server which contains this room. - * @param {String} channel : The channel this room represents. - */ - constructor(server, channel) { - if (!server || !channel) { - throw new Error("Server and channel are required."); - } - channel = toIrcLowerCase(channel); - super(IrcRoom.createId(server, channel), { - domain: server.domain, - channel: channel, - type: channel.indexOf("#") === 0 ? "channel" : "pm" - }); - this.server = server; - this.channel = channel; - } - - getDomain() { - return this.get("domain"); - } - - getServer() { - return this.server; - } - - getChannel() { - return this.get("channel"); - } - - getType() { - return this.get("type"); - } -} - -IrcRoom.fromRemoteRoom = function(server, remoteRoom) { - return new IrcRoom(server, remoteRoom.get("channel")); -}; - -// An IRC room is uniquely identified by a combination of the channel name and the -// IRC network the channel resides on. Space is the delimiter because neither the -// domain nor the channel allows spaces. -IrcRoom.createId = function(server, channel) { - return server.domain + " " + channel; -}; - -module.exports = IrcRoom; diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts new file mode 100644 index 000000000..bc00c7426 --- /dev/null +++ b/src/models/IrcRoom.ts @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//@ts-ignore +import { RemoteRoom } from "matrix-appservice-bridge"; +import { toIrcLowerCase } from "../irc/formatting"; + +export class IrcRoom extends RemoteRoom { + /** + * Construct a new IRC room. + * @constructor + * @param {IrcServer} server : The IRC server which contains this room. + * @param {String} channel : The channel this room represents. + */ + constructor(public readonly server: any, public readonly channel: string) { + // Because `super` must be called first, we convert the case several times. + super(IrcRoom.createId(server, toIrcLowerCase(channel)), { + domain: server.domain, + channel: toIrcLowerCase(channel), + type: channel.indexOf("#") === 0 ? "channel" : "pm" + }); + if (!server || !channel) { + throw new Error("Server and channel are required."); + } + channel = toIrcLowerCase(channel); + } + + getDomain() { + return super.get("domain"); + } + + getServer() { + return this.server; + } + + getChannel() { + return super.get("channel"); + } + + getType() { + return super.get("type"); + } + + public static fromRemoteRoom(server: any, remoteRoom: RemoteRoom) { + return new IrcRoom(server, remoteRoom.get("channel")); + } + + // An IRC room is uniquely identified by a combination of the channel name and the + // IRC network the channel resides on. Space is the delimiter because neither the + // domain nor the channel allows spaces. + public static createId(server: any, channel: string) { + return server.domain + " " + channel; + } +} \ No newline at end of file diff --git a/src/provisioning/Provisioner.js b/src/provisioning/Provisioner.js index 38a190f7f..6a2f545c4 100644 --- a/src/provisioning/Provisioner.js +++ b/src/provisioning/Provisioner.js @@ -1,7 +1,7 @@ /*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping "use strict"; const Promise = require("bluebird"); -const IrcRoom = require("../models/IrcRoom"); +const { IrcRoom } = require("../models/IrcRoom"); const IrcAction = require("../models/IrcAction"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const ConfigValidator = require("matrix-appservice-bridge").ConfigValidator; From 3a3586055d9af1f47a7e263828c5acb6fbdfaabb Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 16 Sep 2019 15:23:09 +0100 Subject: [PATCH 008/350] Add support for Typescript linting --- .ts.eslintrc | 5 ++ package-lock.json | 120 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 6 ++- 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 .ts.eslintrc diff --git a/.ts.eslintrc b/.ts.eslintrc new file mode 100644 index 000000000..c9777cd86 --- /dev/null +++ b/.ts.eslintrc @@ -0,0 +1,5 @@ +{ + "parser": "@typescript-eslint/parser", + "plugins": ["@typescript-eslint"], + "extends": ["plugin:@typescript-eslint/recommended"] +} diff --git a/package-lock.json b/package-lock.json index 120079790..e7228fab7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -141,6 +141,18 @@ "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.7.tgz", "integrity": "sha512-2Y8uPt0/jwjhQ6EiluT0XCri1Dbplr0ZxfFXUz+ye13gaqE8u5gL5ppao1JrUYr9cIip5S6MvQzBS7Kke7U9VA==" }, + "@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", + "dev": true + }, + "@types/json-schema": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.3.tgz", + "integrity": "sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A==", + "dev": true + }, "@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", @@ -151,6 +163,93 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" }, + "@typescript-eslint/eslint-plugin": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.2.0.tgz", + "integrity": "sha512-rOodtI+IvaO8USa6ValYOrdWm9eQBgqwsY+B0PPiB+aSiK6p6Z4l9jLn/jI3z3WM4mkABAhKIqvGIBl0AFRaLQ==", + "dev": true, + "requires": { + "@typescript-eslint/experimental-utils": "2.2.0", + "eslint-utils": "^1.4.2", + "functional-red-black-tree": "^1.0.1", + "regexpp": "^2.0.1", + "tsutils": "^3.17.1" + }, + "dependencies": { + "tsutils": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.17.1.tgz", + "integrity": "sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==", + "dev": true, + "requires": { + "tslib": "^1.8.1" + } + } + } + }, + "@typescript-eslint/experimental-utils": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-2.2.0.tgz", + "integrity": "sha512-IMhbewFs27Frd/ICHBRfIcsUCK213B8MsEUqvKFK14SDPjPR5JF6jgOGPlroybFTrGWpMvN5tMZdXAf+xcmxsA==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "2.2.0", + "eslint-scope": "^5.0.0" + }, + "dependencies": { + "eslint-scope": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.0.0.tgz", + "integrity": "sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + } + } + }, + "@typescript-eslint/parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-2.2.0.tgz", + "integrity": "sha512-0mf893kj9L65O5sA7wP6EoYvTybefuRFavUNhT7w9kjhkdZodoViwVS+k3D+ZxKhvtL7xGtP/y/cNMJX9S8W4A==", + "dev": true, + "requires": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "2.2.0", + "@typescript-eslint/typescript-estree": "2.2.0", + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", + "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", + "dev": true + } + } + }, + "@typescript-eslint/typescript-estree": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.2.0.tgz", + "integrity": "sha512-9/6x23A3HwWWRjEQbuR24on5XIfVmV96cDpGR9671eJv1ebFKHj2sGVVAwkAVXR2UNuhY1NeKS2QMv5P8kQb2Q==", + "dev": true, + "requires": { + "glob": "^7.1.4", + "is-glob": "^4.0.1", + "lodash.unescape": "4.0.1", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1709,11 +1808,26 @@ "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, "is-my-ip-valid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz", @@ -2072,6 +2186,12 @@ "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz", "integrity": "sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==" }, + "lodash.unescape": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", + "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=", + "dev": true + }, "log-symbols": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", diff --git a/package.json b/package.json index b39fbcd40..f15045fe9 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "scripts": { "build": "tsc --project ./tsconfig.json", "test": "BLUEBIRD_DEBUG=1 node --max_old_space_size=3072 node_modules/jasmine/bin/jasmine.js --stop-on-failure=true", - "lint": "eslint --max-warnings 0 src/**/*.js app.js spec", + "lint:js": "eslint --max-warnings 0 src/**/*.js app.js spec", + "lint:ts": "eslint -c .ts.eslintrc --max-warnings 0 src/**/*.ts", + "lint": "npm run lint:js && npm run lint:ts", "check": "npm test && npm run lint", "ci-test": "node --max_old_space_size=3072 node_modules/nyc/bin/nyc.js --report text jasmine", "ci": "npm run lint && npm run ci-test" @@ -44,6 +46,8 @@ "winston-daily-rotate-file": "^3.2.1" }, "devDependencies": { + "@typescript-eslint/eslint-plugin": "^2.2.0", + "@typescript-eslint/parser": "^2.2.0", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", From bf1b1ce14a10f9189ad3d0c94eea108c19b03172 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 17 Sep 2019 00:21:11 +0100 Subject: [PATCH 009/350] Convert IrcClientConfig to Typescript --- src/DataStore.ts | 8 +-- src/bridge/IrcBridge.js | 2 +- src/bridge/MatrixHandler.js | 2 +- src/irc/IrcServer.js | 2 +- src/models/IrcClientConfig.js | 84 --------------------------- src/models/IrcClientConfig.ts | 105 ++++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 91 deletions(-) delete mode 100644 src/models/IrcClientConfig.js create mode 100644 src/models/IrcClientConfig.ts diff --git a/src/DataStore.ts b/src/DataStore.ts index baf67bbf0..efcfdd172 100644 --- a/src/DataStore.ts +++ b/src/DataStore.ts @@ -18,10 +18,11 @@ import * as crypto from "crypto"; import * as fs from "fs"; import {default as Bluebird} from "bluebird"; import { IrcRoom } from "./models/IrcRoom"; +import { IrcClientConfig } from "./models/IrcClientConfig"; // Ignore definition errors for now. //@ts-ignore -import { MatrixRoom, RemoteRoom, MatrixUser, RemoteUser} from "matrix-appservice-bridge"; +import { MatrixRoom, RemoteRoom, MatrixUser, RemoteUser, Logging} from "matrix-appservice-bridge"; interface RoomEntry { id: string; @@ -38,11 +39,10 @@ interface UserFeatures { [name: string]: boolean } -const IrcClientConfig = require("./models/IrcClientConfig"); -const log = require("./logging").get("DataStore"); - export type RoomOrigin = "config"|"provision"|"alias"|"join"; +const log = Logging.get("DataStore"); + export class DataStore { private serverMappings: {[domain: string]: any /*IrcServer*/} = {}; private privateKey: string|null; diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 17707ba81..b960fd21d 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -15,7 +15,7 @@ var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); var IrcUser = require("../models/IrcUser"); const { IrcRoom } = require("../models/IrcRoom"); -var IrcClientConfig = require("../models/IrcClientConfig"); +const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); const { DataStore } = require("../DataStore"); diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index a420f384a..a9ad259b5 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -7,7 +7,7 @@ const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const { IrcRoom } = require("../models/IrcRoom"); const MatrixAction = require("../models/MatrixAction"); const IrcAction = require("../models/IrcAction"); -const IrcClientConfig = require("../models/IrcClientConfig"); +const { IrcClientConfig } = require("../models/IrcClientConfig"); const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const BridgeRequest = require("../models/BridgeRequest"); const toIrcLowerCase = require("../irc/formatting").toIrcLowerCase; diff --git a/src/irc/IrcServer.js b/src/irc/IrcServer.js index 889edbb8f..e06bece02 100644 --- a/src/irc/IrcServer.js +++ b/src/irc/IrcServer.js @@ -3,7 +3,7 @@ */ "use strict"; const logging = require("../logging"); -const IrcClientConfig = require("../models/IrcClientConfig"); +const { IrcClientConfig } = require("../models/IrcClientConfig"); const log = logging.get("IrcServer"); const BridgedClient = require("./BridgedClient"); diff --git a/src/models/IrcClientConfig.js b/src/models/IrcClientConfig.js deleted file mode 100644 index 833b8833e..000000000 --- a/src/models/IrcClientConfig.js +++ /dev/null @@ -1,84 +0,0 @@ -"use strict"; - -/** - * Contains IRC client configuration, mostly set by Matrix users. Used to configure - * IrcUsers. - */ -class IrcClientConfig { - - /** - * Construct an IRC Client Config. - * @param {string} userId The user ID who is configuring this config. - * @param {string} domain The IRC network domain for the IRC client - * @param {Object} configObj Serialised config information if known. - */ - constructor(userId, domain, configObj) { - this.userId = userId; - this.domain = domain; - this._config = configObj || {}; - } - - getDomain() { - return this.domain; - } - - getUserId() { - return this.userId; - } - - setUsername(uname) { - this._config.username = uname; - } - - getUsername() { - return this._config.username; - } - - setPassword(password) { - this._config.password = password; - } - - getPassword() { - return this._config.password; - } - - setDesiredNick(nick) { - this._config.nick = nick; - } - - getDesiredNick() { - return this._config.nick; - } - - setIpv6Address(address) { - this._config.ipv6 = address; - } - - getIpv6Address() { - return this._config.ipv6; - } - - serialize() { - return this._config; - } - - toString() { - let redactedConfig = { - username: this._config.username, - nick: this._config.nick, - ipv6: this._config.ipv6, - password: this._config.password ? '' : undefined, - }; - return this.userId + "=>" + this.domain + "=" + JSON.stringify(redactedConfig); - } -} - -IrcClientConfig.newConfig = function(matrixUser, domain, nick, username, password) { - return new IrcClientConfig(matrixUser ? matrixUser.getId() : null, domain, { - nick: nick, - username: username, - password: password - }); -}; - -module.exports = IrcClientConfig; diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts new file mode 100644 index 000000000..c079c0bbc --- /dev/null +++ b/src/models/IrcClientConfig.ts @@ -0,0 +1,105 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Ignore definition errors for now. +//@ts-ignore +import { MatrixUser } from "matrix-appservice-bridge"; + +interface IrcClientConfigSeralized { + username?: string; + password?: string; + nick?: string; + ipv6?: string; +} + +/** + * Contains IRC client configuration, mostly set by Matrix users. Used to configure + * IrcUsers. + */ +export class IrcClientConfig { + + /** + * Construct an IRC Client Config. + * @param {string} userId The user ID who is configuring this config. + * @param {string} domain The IRC network domain for the IRC client + * @param {Object} configObj Serialised config information if known. + */ + constructor(public userId: string, public domain: string, private config: IrcClientConfigSeralized = {}) { + + } + + public getDomain() { + return this.domain; + } + + public getUserId() { + return this.userId; + } + + public setUsername(uname: string) { + this.config.username = uname; + } + + public getUsername(): string|undefined { + return this.config.username; + } + + public setPassword(password: string) { + this.config.password = password; + } + + public getPassword(): string|undefined { + return this.config.password; + } + + public setDesiredNick(nick: string) { + this.config.nick = nick; + } + + public getDesiredNick(): string|undefined { + return this.config.nick; + } + + public setIpv6Address(address: string) { + this.config.ipv6 = address; + } + + public getIpv6Address(): string|undefined { + return this.config.ipv6; + } + + public serialize() { + return this.config; + } + + public toString() { + let redactedConfig = { + username: this.config.username, + nick: this.config.nick, + ipv6: this.config.ipv6, + password: this.config.password ? '' : undefined, + }; + return this.userId + "=>" + this.domain + "=" + JSON.stringify(redactedConfig); + } + + public static newConfig(matrixUser: MatrixUser, domain: string, nick: string, username: string, password: string) { + return new IrcClientConfig(matrixUser ? matrixUser.getId() : null, domain, { + nick: nick, + username: username, + password: password + }); + } +} From 8b612518e366c1d3640ae62055abd05f75116b96 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 17 Sep 2019 00:26:30 +0100 Subject: [PATCH 010/350] Ensure that password is set --- src/DataStore.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DataStore.ts b/src/DataStore.ts index efcfdd172..4c5f503fe 100644 --- a/src/DataStore.ts +++ b/src/DataStore.ts @@ -526,13 +526,14 @@ export class DataStore { return null; } const clientConfig = new IrcClientConfig(userId, domain, configData); - if (clientConfig.getPassword()) { + const encryptedPass = clientConfig.getPassword(); + if (encryptedPass) { if (!this.privateKey) { throw new Error(`Cannot decrypt password of ${userId} - no private key`); } let decryptedPass = crypto.privateDecrypt( this.privateKey, - new Buffer(clientConfig.getPassword(), 'base64') + new Buffer(encryptedPass, 'base64') ).toString(); // Extract the password by removing the prefixed salt and seperating space decryptedPass = decryptedPass.split(' ')[1]; From 94c3872f4dfe317ff1b8cbba4c49614644254be9 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 17 Sep 2019 00:49:35 +0100 Subject: [PATCH 011/350] Fix linting errors --- .ts.eslintrc | 6 +++++- package-lock.json | 3 ++- src/models/IrcClientConfig.ts | 10 +++++++--- src/models/IrcRoom.ts | 7 +++++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.ts.eslintrc b/.ts.eslintrc index c9777cd86..74fd521d8 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -1,5 +1,9 @@ { "parser": "@typescript-eslint/parser", "plugins": ["@typescript-eslint"], - "extends": ["plugin:@typescript-eslint/recommended"] + "extends": ["plugin:@typescript-eslint/recommended"], + "rules": { + "@typescript-eslint/ban-ts-ignore": 0, + "@typescript-eslint/explicit-function-return-type": 0, + } } diff --git a/package-lock.json b/package-lock.json index 383187e8a..e56545318 100644 --- a/package-lock.json +++ b/package-lock.json @@ -139,7 +139,8 @@ "@types/bluebird": { "version": "3.5.27", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.27.tgz", - "integrity": "sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==" + "integrity": "sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==", + "dev": true }, "@types/chai": { "version": "4.1.7", diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index c079c0bbc..e5b034a0e 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -37,7 +37,10 @@ export class IrcClientConfig { * @param {string} domain The IRC network domain for the IRC client * @param {Object} configObj Serialised config information if known. */ - constructor(public userId: string, public domain: string, private config: IrcClientConfigSeralized = {}) { + constructor( + public userId: string, + public domain: string, + private config: IrcClientConfigSeralized = {}) { } @@ -86,7 +89,7 @@ export class IrcClientConfig { } public toString() { - let redactedConfig = { + const redactedConfig = { username: this.config.username, nick: this.config.nick, ipv6: this.config.ipv6, @@ -95,7 +98,8 @@ export class IrcClientConfig { return this.userId + "=>" + this.domain + "=" + JSON.stringify(redactedConfig); } - public static newConfig(matrixUser: MatrixUser, domain: string, nick: string, username: string, password: string) { + public static newConfig(matrixUser: MatrixUser, domain: string, + nick: string, username: string, password: string) { return new IrcClientConfig(matrixUser ? matrixUser.getId() : null, domain, { nick: nick, username: username, diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index bc00c7426..e90881b37 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -25,6 +25,7 @@ export class IrcRoom extends RemoteRoom { * @param {IrcServer} server : The IRC server which contains this room. * @param {String} channel : The channel this room represents. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(public readonly server: any, public readonly channel: string) { // Because `super` must be called first, we convert the case several times. super(IrcRoom.createId(server, toIrcLowerCase(channel)), { @@ -54,6 +55,8 @@ export class IrcRoom extends RemoteRoom { return super.get("type"); } + // No types for IrcServer yet + // eslint-disable-next-line @typescript-eslint/no-explicit-any public static fromRemoteRoom(server: any, remoteRoom: RemoteRoom) { return new IrcRoom(server, remoteRoom.get("channel")); } @@ -61,7 +64,7 @@ export class IrcRoom extends RemoteRoom { // An IRC room is uniquely identified by a combination of the channel name and the // IRC network the channel resides on. Space is the delimiter because neither the // domain nor the channel allows spaces. - public static createId(server: any, channel: string) { + public static createId(server: {domain: string}, channel: string) { return server.domain + " " + channel; } -} \ No newline at end of file +} From ef96bdbf586adcbd1adbf0610beb7b4660385efd Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:34:17 +0100 Subject: [PATCH 012/350] Create NedbDataStore and model on a generic DataStore interface --- src/datastore/DataStore.ts | 153 +++++++++++++ .../NedbDataStore.ts} | 201 ++++++++---------- 2 files changed, 238 insertions(+), 116 deletions(-) create mode 100644 src/datastore/DataStore.ts rename src/{DataStore.ts => datastore/NedbDataStore.ts} (81%) diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts new file mode 100644 index 000000000..8f3b5b1c6 --- /dev/null +++ b/src/datastore/DataStore.ts @@ -0,0 +1,153 @@ +// Ignore definition errors for now. +//@ts-ignore +import { MatrixRoom, RemoteRoom, MatrixUser} from "matrix-appservice-bridge"; +import {default as Bluebird} from "bluebird"; +import { IrcRoom } from "../models/IrcRoom"; +import { IrcClientConfig } from "../models/IrcClientConfig"; +import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; + +export type RoomOrigin = "config"|"provision"|"alias"|"join"; +export interface RoomEntry { + id: string; + matrix: MatrixRoom; + remote: RemoteRoom; + data: { + origin: RoomOrigin; + }; +} + +export interface ChannelMappings { + [roomId: string]: Array<{networkId: string; channel: string}>; +} + +export interface UserFeatures { + [name: string]: boolean; +} + +export interface DataStore { + setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise; + + /** + * Persists an IRC <--> Matrix room mapping in the database. + * @param {IrcRoom} ircRoom : The IRC room to store. + * @param {MatrixRoom} matrixRoom : The Matrix room to store. + * @param {string} origin : "config" if this mapping is from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via + * aliasing and "join" if it was created during a join. + * @return {Promise} + */ + storeRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, origin: RoomOrigin): Promise; + + /** + * Get an IRC <--> Matrix room mapping from the database. + * @param {string} roomId : The Matrix room ID. + * @param {string} ircDomain : The IRC server domain. + * @param {string} ircChannel : The IRC channel. + * @param {string} origin : (Optional) "config" if this mapping was from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via aliasing and + * "join" if it was created during a join. + * @return {Promise} A promise which resolves to a room entry, or null if one is not found. + */ + getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise; + + /** + * Get all Matrix <--> IRC room mappings from the database. + * @return {Promise} A promise which resolves to a map: + * $roomId => [{networkId: 'server #channel1', channel: '#channel2'} , ...] + */ + getAllChannelMappings(): Promise; + + /** + * Get provisioned IRC <--> Matrix room mappings from the database where + * the matrix room ID is roomId. + * @param {string} roomId : The Matrix room ID. + * @return {Promise} A promise which resolves to a list + * of entries. + */ + getProvisionedMappings(roomId: string): Bluebird; + + /** + * Remove an IRC <--> Matrix room mapping from the database. + * @param {string} roomId : The Matrix room ID. + * @param {string} ircDomain : The IRC server domain. + * @param {string} ircChannel : The IRC channel. + * @param {string} origin : "config" if this mapping was from the config yaml, + * "provision" if this mapping was provisioned, "alias" if it was created via + * aliasing and "join" if it was created during a join. + * @return {Promise} + */ + removeRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise; + + /** + * Retrieve a list of IRC rooms for a given room ID. + * @param {string} roomId : The room ID to get mapped IRC channels. + * @return {Promise>} A promise which resolves to a list of + * rooms. + */ + getIrcChannelsForRoomId(roomId: string): Promise; + + + /** + * Retrieve a list of IRC rooms for a given list of room IDs. This is significantly + * faster than calling getIrcChannelsForRoomId for each room ID. + * @param {string[]} roomIds : The room IDs to get mapped IRC channels. + * @return {Promise>} A promise which resolves to a map of + * room ID to an array of IRC rooms. + */ + getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: IrcRoom[]}>; + + /** + * Retrieve a list of Matrix rooms for a given server and channel. + * @param {IrcServer} server : The server to get rooms for. + * @param {string} channel : The channel to get mapped rooms for. + * @return {Promise>} A promise which resolves to a list of rooms. + */ + getMatrixRoomsForChannel(server: IrcServer, channel: string): Promise>; + + getMappingsForChannelByOrigin(server: IrcServer, channel: string, + origin: RoomOrigin|RoomOrigin[], allowUnset: boolean): Promise; + + getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string}>; + + setModeForRoom(roomId: string, mode: string, enabled: boolean): Promise; + + setPmRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise; + + getMatrixPmRoom(realUserId: string, virtualUserId: string): Promise; + + getTrackedChannelsForServer(domain: string): Promise; + + getRoomIdsFromConfig(): Promise; + + removeConfigMappings(): Promise; + + getIpv6Counter(): Promise; + + setIpv6Counter(counter: number): Promise; + + getAdminRoomById(roomId: string): Promise; + + storeAdminRoom(room: MatrixRoom, userId: string): Promise; + + upsertRoomStoreEntry(entry: RoomEntry): Promise; + + getAdminRoomByUserId(userId: string): Promise; + + storeMatrixUser(matrixUser: MatrixUser): Promise; + + getIrcClientConfig(userId: string, domain: string): Promise; + + storeIrcClientConfig(config: IrcClientConfig): Promise; + + getMatrixUserByLocalpart(localpart: string): Promise; + + getUserFeatures(userId: string): Promise; + + storeUserFeatures(userId: string, features: UserFeatures): Promise; + + storePass(userId: string, domain: string, pass: string): Promise; + + removePass(userId: string, domain: string): Promise; + + getMatrixUserByUsername(domain: string, username: string): Promise; +} diff --git a/src/DataStore.ts b/src/datastore/NedbDataStore.ts similarity index 81% rename from src/DataStore.ts rename to src/datastore/NedbDataStore.ts index 4c5f503fe..42cbed00f 100644 --- a/src/DataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -14,40 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -import * as crypto from "crypto"; -import * as fs from "fs"; import {default as Bluebird} from "bluebird"; -import { IrcRoom } from "./models/IrcRoom"; -import { IrcClientConfig } from "./models/IrcClientConfig"; +import { IrcRoom } from "../models/IrcRoom"; +import { IrcClientConfig } from "../models/IrcClientConfig" +import * as logging from "../logging"; // Ignore definition errors for now. //@ts-ignore -import { MatrixRoom, RemoteRoom, MatrixUser, RemoteUser, Logging} from "matrix-appservice-bridge"; +import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom} from "matrix-appservice-bridge"; +import { DataStore, RoomOrigin, ChannelMappings, RoomEntry, UserFeatures } from "./DataStore"; +import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; +import { StringCrypto } from "./StringCrypto"; -interface RoomEntry { - id: string; - matrix: MatrixRoom; - remote: RemoteRoom; - data: any; -} - -interface ChannelMappings { - [roomId: string]: Array<{networkId: string, channel: string}> -} - -interface UserFeatures { - [name: string]: boolean -} - -export type RoomOrigin = "config"|"provision"|"alias"|"join"; +const log = logging.get("NeDBDataStore"); -const log = Logging.get("DataStore"); - -export class DataStore { - private serverMappings: {[domain: string]: any /*IrcServer*/} = {}; - private privateKey: string|null; +export class NeDBDataStore implements DataStore { + private serverMappings: {[domain: string]: IrcServer} = {}; + private cryptoStore?: StringCrypto; constructor( + // eslint-disable-next-line @typescript-eslint/no-explicit-any private userStore: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any private roomStore: any, pkeyPath: string, private bridgeDomain: string) { @@ -60,7 +47,7 @@ export class DataStore { log.info("Indexes checked on '%s' for store.", fieldName); }; }; - + // add some indexes this.roomStore.db.ensureIndex({ fieldName: "id", @@ -88,40 +75,19 @@ export class DataStore { sparse: false }, errLog("user id")); - this.privateKey = null; - if (pkeyPath) { - try { - this.privateKey = fs.readFileSync(pkeyPath, "utf8").toString(); - - // Test whether key is a valid PEM key (publicEncrypt does internal validation) - try { - crypto.publicEncrypt( - this.privateKey, - new Buffer("This is a test!") - ); - } - catch (err) { - log.error(`Failed to validate private key: (${err.message})`); - throw err; - } - - log.info(`Private key loaded from ${pkeyPath} - IRC password encryption enabled.`); - } - catch (err) { - log.error(`Could not load private key ${err.message}.`); - throw err; - } + this.cryptoStore = new StringCrypto(); + this.cryptoStore.load(pkeyPath); } // Cache as many mappings as possible for hot paths like message sending. - + // TODO: cache IRC channel -> [room_id] mapping (only need to remove them in // removeRoom() which is infrequent) // TODO: cache room_id -> [#channel] mapping (only need to remove them in // removeRoom() which is infrequent) } - public async setServerFromConfig(server: any, serverConfig: any): Promise { + public async setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise { this.serverMappings[server.domain] = server; for (const channel of Object.keys(serverConfig.mappings)) { @@ -131,7 +97,7 @@ export class DataStore { await this.storeRoom(ircRoom, mxRoom, "config"); } } - + // Some kinds of users may have the same user_id prefix so will cause ident code to hit // getMatrixUserByUsername hundreds of times which can be slow: // https://github.com/matrix-org/matrix-appservice-irc/issues/404 @@ -166,11 +132,11 @@ export class DataStore { log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin); - const mappingId = DataStore.createMappingId(matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel); + const mappingId = NeDBDataStore.createMappingId(matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel); await this.roomStore.linkRooms(matrixRoom, ircRoom, { origin: origin }, mappingId); - }; + } /** * Get an IRC <--> Matrix room mapping from the database. @@ -182,19 +148,20 @@ export class DataStore { * "join" if it was created during a join. * @return {Promise} A promise which resolves to a room entry, or null if one is not found. */ - public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise { - if (typeof origin !== "undefined" && typeof origin !== "string") { + public async getRoom(roomId: string, ircDomain: string, + ircChannel: string, origin?: RoomOrigin): Promise { + if (origin && typeof origin !== "string") { throw new Error(`If defined, origin must be a string = "config"|"provision"|"alias"|"join"`); } - const mappingId = DataStore.createMappingId(roomId, ircDomain, ircChannel); + const mappingId = NeDBDataStore.createMappingId(roomId, ircDomain, ircChannel); return this.roomStore.getEntryById(mappingId).then( (entry: RoomEntry) => { if (origin && entry && origin !== entry.data.origin) { return null; } return entry; - }); + }); } /** @@ -213,13 +180,13 @@ export class DataStore { const mappings: ChannelMappings = {}; - entries.forEach((e: any) => { + entries.forEach((e: { matrix_id: string; remote: {domain: string; channel: string}}) => { // drop unknown irc networks in the database if (!this.serverMappings[e.remote.domain]) { return; } if (!mappings[e.matrix_id]) { - mappings[e.matrix_id] = new Array(); + mappings[e.matrix_id] = []; } mappings[e.matrix_id].push({ networkId: this.serverMappings[e.remote.domain].getNetworkId(), @@ -261,7 +228,7 @@ export class DataStore { } return await this.roomStore.delete({ - id: DataStore.createMappingId(roomId, ircDomain, ircChannel), + id: NeDBDataStore.createMappingId(roomId, ircDomain, ircChannel), 'data.origin': origin }); } @@ -275,9 +242,9 @@ export class DataStore { public async getIrcChannelsForRoomId(roomId: string): Promise { return this.roomStore.getLinkedRemoteRooms(roomId).then((remoteRooms: RemoteRoom[]) => { return remoteRooms.filter((remoteRoom) => { - return Boolean(this.serverMappings[remoteRoom.get("domain")]); + return Boolean(this.serverMappings[remoteRoom.get("domain") as string]); }).map((remoteRoom) => { - let server = this.serverMappings[remoteRoom.get("domain")]; + const server = this.serverMappings[remoteRoom.get("domain") as string]; return IrcRoom.fromRemoteRoom(server, remoteRoom); }); }); @@ -291,14 +258,16 @@ export class DataStore { * room ID to an array of IRC rooms. */ public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: IrcRoom[]}> { - const roomIdToRemoteRooms: {[roomId: string]: RemoteRoom[]} = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); + const roomIdToRemoteRooms: { + [roomId: string]: IrcRoom[]; + } = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); for (const roomId of Object.keys(roomIdToRemoteRooms)) { // filter out rooms with unknown IRC servers and // map RemoteRooms to IrcRooms roomIdToRemoteRooms[roomId] = roomIdToRemoteRooms[roomId].filter((remoteRoom) => { - return Boolean(this.serverMappings[remoteRoom.get("domain")]); + return Boolean(this.serverMappings[remoteRoom.get("domain") as string]); }).map((remoteRoom) => { - const server = this.serverMappings[remoteRoom.get("domain")]; + const server = this.serverMappings[remoteRoom.get("domain") as string]; return IrcRoom.fromRemoteRoom(server, remoteRoom); }); } @@ -311,14 +280,15 @@ export class DataStore { * @param {string} channel : The channel to get mapped rooms for. * @return {Promise>} A promise which resolves to a list of rooms. */ - public async getMatrixRoomsForChannel(server: any, channel: string): Promise> { + public async getMatrixRoomsForChannel(server: IrcServer, channel: string): Promise { const ircRoom = new IrcRoom(server, channel); return await this.roomStore.getLinkedMatrixRooms( IrcRoom.createId(ircRoom.getServer(), ircRoom.getChannel()) ); } - public async getMappingsForChannelByOrigin(server: any, channel: string, origin: RoomOrigin|RoomOrigin[], allowUnset: boolean) { + public async getMappingsForChannelByOrigin(server: IrcServer, channel: string, + origin: RoomOrigin|RoomOrigin[], allowUnset: boolean) { if (typeof origin === "string") { origin = [origin]; } @@ -340,27 +310,27 @@ export class DataStore { }); } - public async getModesForChannel (server: any, channel: string): Promise<{[id: string]: string}> { + public async getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string}> { log.info("getModesForChannel (server=%s, channel=%s)", server.domain, channel ); const remoteId = IrcRoom.createId(server, channel); return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: RoomEntry[]) => { - const mapping: {[id: string]: string} = {}; + const mapping: {[id: string]: string[]} = {}; entries.forEach((entry) => { - mapping[entry.matrix.getId()] = entry.remote.get("modes") || []; + mapping[entry.matrix.getId()] = entry.remote.get("modes") as string[] || []; }); return mapping; }); } - public async setModeForRoom(roomId: string, mode: string, enabled: boolean = true): Promise { + public async setModeForRoom(roomId: string, mode: string, enabled = true): Promise { log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", mode, roomId, enabled ); const entries: RoomEntry[] = await this.roomStore.getEntriesByMatrixId(roomId); for (const entry of entries) { - const modes = entry.remote.get("modes") || []; + const modes = entry.remote.get("modes") as string[] || []; const hasMode = modes.includes(mode); if (hasMode === enabled) { @@ -379,19 +349,20 @@ export class DataStore { } } - public async setPmRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise { + public async setPmRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, + userId: string, virtualUserId: string): Promise { log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)", matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId, virtualUserId); - + await this.roomStore.linkRooms(matrixRoom, ircRoom, { real_user_id: userId, virtual_user_id: virtualUserId - }, DataStore.createPmId(userId, virtualUserId)); + }, NeDBDataStore.createPmId(userId, virtualUserId)); } - + public async getMatrixPmRoom(realUserId: string, virtualUserId: string) { - const id = DataStore.createPmId(realUserId, virtualUserId); + const id = NeDBDataStore.createPmId(realUserId, virtualUserId); const entry = await this.roomStore.getEntryById(id); if (!entry) { return null; @@ -404,7 +375,7 @@ export class DataStore { const channels: string[] = []; entries.forEach((e) => { const r = e.remote; - const server = this.serverMappings[r.get("domain")]; + const server = this.serverMappings[r.get("domain") as string]; if (!server) { return; } @@ -453,7 +424,7 @@ export class DataStore { config.set("ipv6_counter", counter); await this.userStore.setRemoteUser(config); } - + /** * Retrieve a stored admin room based on the room's ID. * @param {String} roomId : The room ID of the admin room. @@ -483,7 +454,7 @@ export class DataStore { log.info("storeAdminRoom (id=%s, user_id=%s)", room.getId(), userId); room.set("admin_id", userId); await this.roomStore.upsertEntry({ - id: DataStore.createAdminId(userId), + id: NeDBDataStore.createAdminId(userId), matrix: room, }); } @@ -491,20 +462,20 @@ export class DataStore { public async upsertRoomStoreEntry(entry: RoomEntry): Promise { await this.roomStore.upsertEntry(entry); } - - public async getAdminRoomByUserId(userId: string): Promise { - const entry = await this.roomStore.getEntryById(DataStore.createAdminId(userId)); + + public async getAdminRoomByUserId(userId: string): Promise { + const entry = await this.roomStore.getEntryById(NeDBDataStore.createAdminId(userId)); if (!entry) { return null; } return entry.matrix; } - + public async storeMatrixUser(matrixUser: MatrixUser): Promise { await this.userStore.setMatrixUser(matrixUser); } - public async getIrcClientConfig(userId: string, domain: string): Promise /*IrcClientConfig*/ { + public async getIrcClientConfig(userId: string, domain: string): Promise { const matrixUser = await this.userStore.getMatrixUser(userId); if (!matrixUser) { return null; @@ -515,7 +486,7 @@ export class DataStore { } // map back from _ to . Object.keys(userConfig).forEach(function(domainWithUnderscores) { - let actualDomain = domainWithUnderscores.replace(/_/g, "."); + const actualDomain = domainWithUnderscores.replace(/_/g, "."); if (actualDomain !== domainWithUnderscores) { // false for 'localhost' userConfig[actualDomain] = userConfig[domainWithUnderscores]; delete userConfig[domainWithUnderscores]; @@ -528,41 +499,37 @@ export class DataStore { const clientConfig = new IrcClientConfig(userId, domain, configData); const encryptedPass = clientConfig.getPassword(); if (encryptedPass) { - if (!this.privateKey) { + if (!this.cryptoStore) { throw new Error(`Cannot decrypt password of ${userId} - no private key`); } - let decryptedPass = crypto.privateDecrypt( - this.privateKey, - new Buffer(encryptedPass, 'base64') - ).toString(); - // Extract the password by removing the prefixed salt and seperating space - decryptedPass = decryptedPass.split(' ')[1]; + const decryptedPass = this.cryptoStore.decrypt(encryptedPass); clientConfig.setPassword(decryptedPass); } return clientConfig; } - - public async getMatrixUserByLocalpart(localpart: string): Promise { + + public async getMatrixUserByLocalpart(localpart: string): Promise { return await this.userStore.getMatrixUser(`@${localpart}:${this.bridgeDomain}`); } - public async storeIrcClientConfig(config: any /*IrcConfig*/) { - let user = await this.userStore.getMatrixUser(config.getUserId()); + public async storeIrcClientConfig(config: IrcClientConfig) { + const userId = config.getUserId(); + if (!userId) { + throw Error("No userId defined in config"); + } + let user = await this.userStore.getMatrixUser(userId); if (!user) { - user = new MatrixUser(config.getUserId()); + user = new MatrixUser(userId); } const userConfig = user.get("client_config") || {}; - if (config.getPassword()) { - if (!this.privateKey) { + const password = config.getPassword(); + if (password) { + if (!this.cryptoStore) { throw new Error( 'Cannot store plaintext passwords' ); } - const salt = crypto.randomBytes(16).toString('base64'); - const encryptedPass = crypto.publicEncrypt( - this.privateKey, - new Buffer(salt + ' ' + config.getPassword()) - ).toString('base64'); + const encryptedPass = this.cryptoStore.encrypt(password); // Store the encrypted password, ready for the db config.setPassword(encryptedPass); } @@ -570,7 +537,7 @@ export class DataStore { user.set("client_config", userConfig); await this.userStore.setMatrixUser(user); } - + public async getUserFeatures(userId: string): Promise { const matrixUser = await this.userStore.getMatrixUser(userId); return matrixUser ? (matrixUser.get("features") || {}) : {}; @@ -586,7 +553,7 @@ export class DataStore { } public async storePass(userId: string, domain: string, pass: string) { - let config = await this.getIrcClientConfig(userId, domain); + const config = await this.getIrcClientConfig(userId, domain); if (!config) { throw new Error(`${userId} does not have an IRC client configured for ${domain}`); } @@ -596,11 +563,13 @@ export class DataStore { public async removePass(userId: string, domain: string) { const config = await this.getIrcClientConfig(userId, domain); - config.setPassword(undefined); - await this.storeIrcClientConfig(config); + if (config) { + config.setPassword(); + await this.storeIrcClientConfig(config); + } } - public async getMatrixUserByUsername(domain: string, username: string) { + public async getMatrixUserByUsername(domain: string, username: string): Promise { const domainKey = domain.replace(/\./g, "_"); const matrixUsers = await this.userStore.getByMatrixData({ ["client_config." + domainKey + ".username"]: username @@ -614,18 +583,18 @@ export class DataStore { } return matrixUsers[0]; } - + private static createPmId(userId: string, virtualUserId: string) { // space as delimiter as none of these IDs allow spaces. return "PM_" + userId + " " + virtualUserId; // clobber based on this. } - + private static createAdminId(userId: string) { return "ADMIN_" + userId; // clobber based on this. } - + private static createMappingId(roomId: string, ircDomain: string, ircChannel: string) { // space as delimiter as none of these IDs allow spaces. return roomId + " " + ircDomain + " " + ircChannel; // clobber based on this } -} \ No newline at end of file +} From 57db9332a2bcede3f1b622f5af36e26ee38162e5 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:37:05 +0100 Subject: [PATCH 013/350] Convert IrcServer to Typescript --- spec/unit/IrcServer.spec.js | 2 +- src/bridge/IrcBridge.js | 15 +- src/irc/IrcServer.js | 582 ------------------------------- src/irc/IrcServer.ts | 672 ++++++++++++++++++++++++++++++++++++ src/main.js | 2 +- src/models/IrcRoom.ts | 11 +- 6 files changed, 689 insertions(+), 595 deletions(-) delete mode 100644 src/irc/IrcServer.js create mode 100644 src/irc/IrcServer.ts diff --git a/spec/unit/IrcServer.spec.js b/spec/unit/IrcServer.spec.js index ab359b3ae..33b04befa 100644 --- a/spec/unit/IrcServer.spec.js +++ b/spec/unit/IrcServer.spec.js @@ -1,5 +1,5 @@ "use strict"; -const IrcServer = require("../../lib/irc/IrcServer"); +const { IrcServer } = require("../../lib/irc/IrcServer"); const extend = require("extend"); describe("IrcServer", function() { describe("getNick", function() { diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index b960fd21d..edcd08e6b 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -9,7 +9,7 @@ var MatrixHandler = require("./MatrixHandler.js"); var MemberListSyncer = require("./MemberListSyncer.js"); var IdentGenerator = require("../irc/IdentGenerator.js"); var Ipv6Generator = require("../irc/Ipv6Generator.js"); -var IrcServer = require("../irc/IrcServer.js"); +const { IrcServer } = require("../irc/IrcServer.js"); var ClientPool = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); @@ -18,7 +18,8 @@ const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); -const { DataStore } = require("../DataStore"); +const { NeDBDataStore } = require("../datastore/NedbDataStore"); +const { PgDataStore } = require("../datastore/postgres/PgDataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, @@ -318,11 +319,13 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; + this._dataStore = new NeDBDataStore( + this._bridge.getUserStore(), + this._bridge.getRoomStore(), + pkeyPath, + this.config.homeserver.domain, + ); - this._dataStore = new DataStore( - this._bridge.getUserStore(), this._bridge.getRoomStore(), pkeyPath, - this.config.homeserver.domain - ); yield this._dataStore.removeConfigMappings(); this._identGenerator = new IdentGenerator(this._dataStore); this._ipv6Generator = new Ipv6Generator(this._dataStore); diff --git a/src/irc/IrcServer.js b/src/irc/IrcServer.js deleted file mode 100644 index e06bece02..000000000 --- a/src/irc/IrcServer.js +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Represents a single IRC server from config.yaml - */ -"use strict"; -const logging = require("../logging"); -const { IrcClientConfig } = require("../models/IrcClientConfig"); -const log = logging.get("IrcServer"); -const BridgedClient = require("./BridgedClient"); - -const GROUP_ID_REGEX = /^\+\S+:\S+$/; - -/** - * Construct a new IRC Server. - * @constructor - * @param {string} domain : The IRC network address - * @param {Object} serverConfig : The config options for this network. - * @param {string} homeserverDomain : The domain of the homeserver - * e.g "matrix.org" - * @param {Number} expiryTimeSeconds : How old a matrix message can be - * before it is considered 'expired' and not sent to IRC. If 0, messages - * will never expire. - */ -function IrcServer(domain, serverConfig, homeserverDomain, expiryTimeSeconds) { - this.domain = domain; - this.config = serverConfig; - - this._addresses = serverConfig.additionalAddresses; - if (!this._addresses) { - this._addresses = []; - } - this._addresses.push(domain); - this._homeserverDomain = homeserverDomain; - this._expiryTimeSeconds = expiryTimeSeconds; - - if (this.config.dynamicChannels.groupId !== undefined && - this.config.dynamicChannels.groupId.trim() !== "") { - this._groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; - if (!this._groupIdValid) { - log.warn( -`${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` - ); - } - } - else { - this._groupIdValid = false; - } -} - -/** - * Get how old a matrix message can be (in seconds) before it is considered - * 'expired' and not sent to IRC. - * @return {Number} The number of seconds. If 0, they never expire. - */ -IrcServer.prototype.getExpiryTimeSeconds = function() { - return this._expiryTimeSeconds || 0; -} - -/** - * Get a string that represents the human-readable name for a server. - * @return {string} this.config.name if truthy, otherwise it will return - * an empty string. - */ -IrcServer.prototype.getReadableName = function() { - let name = this.config.name; - if (name) { - return name; - } - - return ''; -} - -/** - * Return a randomised server domain from the default and additional addresses. - * @return {string} - */ -IrcServer.prototype.randomDomain = function() { - return this._addresses[ - Math.floor((Math.random() * 1000) % this._addresses.length) - ]; -} - -/** - * Returns the network ID of this server, which should be unique across all - * IrcServers on the bridge. Defaults to the domain of this IrcServer. - * @return {string} this.config.networkId || this.domain - */ -IrcServer.prototype.getNetworkId = function() { - return this.config.networkId || this.domain; -} - -/** - * Returns whether the server is configured to wait getQuitDebounceDelayMs before - * parting a user that has disconnected due to a net-split. - * @return {Boolean} this.config.quitDebounce.enabled. - */ -IrcServer.prototype.shouldDebounceQuits = function() { - return this.config.quitDebounce.enabled; -} - -/** - * Get the minimum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If the user rejoins a channel before bridging - * the quit to a leave, the leave will not be sent. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMinMs = function() { - return this.config.quitDebounce.delayMinMs; -} - -/** - * Get the maximum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If a leave is bridged, it will occur at a - * random time between delayMinMs (see above) delayMaxMs. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMaxMs = function() { - return this.config.quitDebounce.delayMaxMs; -} - -/** - * Get the rate of maximum quits received per second before a net-split is - * detected. If the rate of quits received becomes higher that this value, - * a net split is considered ongoing. - * @return {number} - */ -IrcServer.prototype.getDebounceQuitsPerSecond = function() { - return this.config.quitDebounce.quitsPerSecond; -} - -/** - * Get a map that converts IRC user modes to Matrix power levels. - * @return {Object} - */ -IrcServer.prototype.getModePowerMap = function() { - return this.config.modePowerMap || {}; -} - -IrcServer.prototype.getHardCodedRoomIds = function() { - var roomIds = new Set(); - var channels = Object.keys(this.config.mappings); - channels.forEach((chan) => { - this.config.mappings[chan].forEach((roomId) => { - roomIds.add(roomId); - }); - }); - return Array.from(roomIds.keys()); -}; - -IrcServer.prototype.shouldSendConnectionNotices = function() { - return this.config.sendConnectionMessages; -}; - -IrcServer.prototype.isBotEnabled = function() { - return this.config.botConfig.enabled; -}; - -IrcServer.prototype.getUserModes = function() { - return this.config.ircClients.userModes || ""; -} - -IrcServer.prototype.getJoinRule = function() { - return this.config.dynamicChannels.joinRule; -}; - -IrcServer.prototype.areGroupsEnabled = function() { - return this._groupIdValid; -}; - -IrcServer.prototype.getGroupId = function() { - return this.config.dynamicChannels.groupId; -}; - -IrcServer.prototype.shouldFederatePMs = function() { - return this.config.privateMessages.federate; -}; - -IrcServer.prototype.getMemberListFloodDelayMs = function() { - return this.config.membershipLists.floodDelayMs; -}; - -IrcServer.prototype.shouldFederate = function() { - return this.config.dynamicChannels.federate; -}; -IrcServer.prototype.forceRoomVersion = function() { - return this.config.dynamicChannels.roomVersion; -}; - -IrcServer.prototype.getPort = function() { - return this.config.port; -}; - -IrcServer.prototype.isInWhitelist = function(userId) { - return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; -}; - -IrcServer.prototype.getCA = function() { - return this.config.ca; -}; - -IrcServer.prototype.useSsl = function() { - return Boolean(this.config.ssl); -}; - -IrcServer.prototype.useSslSelfSigned = function() { - return Boolean(this.config.sslselfsign); -}; - -IrcServer.prototype.useSasl = function() { - return Boolean(this.config.sasl); -}; - -IrcServer.prototype.allowExpiredCerts = function() { - return Boolean(this.config.allowExpiredCerts); -}; - -IrcServer.prototype.getIdleTimeout = function() { - return this.config.ircClients.idleTimeout; -}; - -IrcServer.prototype.getReconnectIntervalMs = function() { - return this.config.ircClients.reconnectIntervalMs; -}; - -IrcServer.prototype.getConcurrentReconnectLimit = function() { - return this.config.ircClients.concurrentReconnectLimit; -}; - -IrcServer.prototype.getMaxClients = function() { - return this.config.ircClients.maxClients; -}; - -IrcServer.prototype.shouldPublishRooms = function() { - return this.config.dynamicChannels.published; -}; - -IrcServer.prototype.allowsNickChanges = function() { - return this.config.ircClients.allowNickChanges; -}; - -IrcServer.prototype.getBotNickname = function() { - return this.config.botConfig.nick; -}; - -IrcServer.prototype.createBotIrcClientConfig = function(username) { - return IrcClientConfig.newConfig( - null, this.domain, this.config.botConfig.nick, username, - this.config.botConfig.password - ); -}; - -IrcServer.prototype.getIpv6Prefix = function() { - return this.config.ircClients.ipv6.prefix; -}; - -IrcServer.prototype.getIpv6Only = function() { - return this.config.ircClients.ipv6.only; -}; - -IrcServer.prototype.getLineLimit = function() { - return this.config.ircClients.lineLimit; -}; - -IrcServer.prototype.getJoinAttempts = function() { - return this.config.matrixClients.joinAttempts; -}; - -IrcServer.prototype.isExcludedChannel = function(channel) { - return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; -}; - -IrcServer.prototype.hasInviteRooms = function() { - return ( - this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" - ); -}; - -// check if this server dynamically create rooms with aliases. -IrcServer.prototype.createsDynamicAliases = function() { - return ( - this.config.dynamicChannels.enabled && - this.config.dynamicChannels.createAlias - ); -}; - -// check if this server dynamically creates rooms which are joinable via an alias only. -IrcServer.prototype.createsPublicAliases = function() { - return ( - this.createsDynamicAliases() && - this.getJoinRule() === "public" - ); -}; - -IrcServer.prototype.allowsPms = function() { - return this.config.privateMessages.enabled; -}; - -IrcServer.prototype.shouldSyncMembershipToIrc = function(kind, roomId) { - return this._shouldSyncMembership(kind, roomId, true); -}; - -IrcServer.prototype.shouldSyncMembershipToMatrix = function(kind, channel) { - return this._shouldSyncMembership(kind, channel, false); -}; - -IrcServer.prototype._shouldSyncMembership = function(kind, identifier, toIrc) { - if (["incremental", "initial"].indexOf(kind) === -1) { - throw new Error("Bad kind: " + kind); - } - if (!this.config.membershipLists.enabled) { - return false; - } - var shouldSync = this.config.membershipLists.global[ - toIrc ? "matrixToIrc" : "ircToMatrix" - ][kind]; - - if (!identifier) { - return shouldSync; - } - - // check for specific rules for the room id / channel - if (toIrc) { - // room rules clobber global rules - this.config.membershipLists.rooms.forEach(function(r) { - if (r.room === identifier && r.matrixToIrc) { - shouldSync = r.matrixToIrc[kind]; - } - }); - } - else { - // channel rules clobber global rules - this.config.membershipLists.channels.forEach(function(chan) { - if (chan.channel === identifier && chan.ircToMatrix) { - shouldSync = chan.ircToMatrix[kind]; - } - }); - } - - return shouldSync; -}; - -IrcServer.prototype.shouldJoinChannelsIfNoUsers = function() { - return this.config.botConfig.joinChannelsIfNoUsers; -}; - -IrcServer.prototype.isMembershipListsEnabled = function() { - return this.config.membershipLists.enabled; -}; - -IrcServer.prototype.getUserLocalpart = function(nick) { - // the template is just a literal string with special vars; so find/replace - // the vars and strip the @ - var uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); - return uid.replace(/\$NICK/g, nick).substring(1); -}; - -IrcServer.prototype.claimsUserId = function(userId) { - // the server claims the given user ID if the ID matches the user ID template. - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(userId); -}; - -IrcServer.prototype.getNickFromUserId = function(userId) { - // extract the nick from the given user ID - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*?)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(userId); - if (!match) { - return null; - } - return match[1]; -}; - -IrcServer.prototype.getUserIdFromNick = function(nick) { - var template = this.config.matrixClients.userTemplate; - return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + - ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getDisplayNameFromNick = function(nick) { - var template = this.config.matrixClients.displayName; - var displayName = template.replace(/\$NICK/g, nick); - displayName = displayName.replace(/\$SERVER/g, this.domain); - return displayName; -}; - -IrcServer.prototype.claimsAlias = function(alias) { - // the server claims the given alias if the alias matches the alias template - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "#(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(alias); -}; - -IrcServer.prototype.getChannelFromAlias = function(alias) { - // extract the channel from the given alias - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "([^:]*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(alias); - if (!match) { - return null; - } - log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); - return match[1]; -}; - -IrcServer.prototype.getAliasFromChannel = function(channel) { - var template = this.config.dynamicChannels.aliasTemplate; - return template.replace(/\$CHANNEL/, channel) + ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getNick = function(userId, displayName) { - const illegalChars = BridgedClient.illegalCharactersRegex; - let localpart = userId.substring(1).split(":")[0]; - localpart = localpart.replace(illegalChars, ""); - displayName = displayName ? displayName.replace(illegalChars, "") : undefined; - const display = [displayName, localpart].find((n) => Boolean(n)); - if (!display) { - throw new Error("Could not get nick for user, all characters were invalid"); - } - const template = this.config.ircClients.nickTemplate; - let nick = template.replace(/\$USERID/g, userId); - nick = nick.replace(/\$LOCALPART/g, localpart); - nick = nick.replace(/\$DISPLAY/g, display); - return nick; -}; - -IrcServer.prototype.getAliasRegex = function() { - return templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -IrcServer.prototype.getUserRegex = function() { - return templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$NICK": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -function templateToRegex(template, literalVars, regexVars, suffix) { - // The 'template' is a literal string with some special variables which need - // to be find/replaced. - var regex = template; - Object.keys(literalVars).forEach(function(varPlaceholder) { - regex = regex.replace( - new RegExp(escapeRegExp(varPlaceholder), 'g'), - literalVars[varPlaceholder] - ); - }); - - // at this point the template is still a literal string, so escape it before - // applying the regex vars. - regex = escapeRegExp(regex); - // apply regex vars - Object.keys(regexVars).forEach(function(varPlaceholder) { - regex = regex.replace( - // double escape, because we bluntly escaped the entire string before - // so our match is now escaped. - new RegExp(escapeRegExp(escapeRegExp(varPlaceholder)), 'g'), - regexVars[varPlaceholder] - ); - }); - - suffix = suffix || ""; - return regex + suffix; -} - -function escapeRegExp(string) { - // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions - return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -IrcServer.DEFAULT_CONFIG = { - sendConnectionMessages: true, - quitDebounce: { - enabled: false, - quitsPerSecond: 5, - delayMinMs: 3600000, // 1h - delayMaxMs: 7200000, // 2h - }, - botConfig: { - nick: "appservicebot", - joinChannelsIfNoUsers: true, - enabled: true - }, - privateMessages: { - enabled: true, - exclude: [], - federate: true - }, - dynamicChannels: { - enabled: false, - published: true, - createAlias: true, - joinRule: "public", - federate: true, - aliasTemplate: "#irc_$SERVER_$CHANNEL", - whitelist: [], - exclude: [] - }, - mappings: {}, - matrixClients: { - userTemplate: "@$SERVER_$NICK", - displayName: "$NICK (IRC)", - joinAttempts: -1, - }, - ircClients: { - nickTemplate: "M-$DISPLAY", - maxClients: 30, - idleTimeout: 172800, - reconnectIntervalMs: 5000, - concurrentReconnectLimit: 50, - allowNickChanges: false, - ipv6: {only: false}, - lineLimit: 3 - }, - membershipLists: { - enabled: false, - floodDelayMs: 10000, // 10s - global: { - ircToMatrix: { - initial: false, - incremental: false - }, - matrixToIrc: { - initial: false, - incremental: false - } - }, - channels: [], - rooms: [] - } -}; - -module.exports = IrcServer; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts new file mode 100644 index 000000000..40cf1ef92 --- /dev/null +++ b/src/irc/IrcServer.ts @@ -0,0 +1,672 @@ + +import * as logging from "../logging"; +import * as BridgedClient from "./BridgedClient"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +const log = logging.get("IrcServer"); +const GROUP_ID_REGEX = /^\+\S+:\S+$/ + +type MembershipSyncKind = "incremental"|"initial"; + +/* + * Represents a single IRC server from config.yaml + */ +export class IrcServer { + private addresses: string[]; + private groupIdValid: boolean; + /** + * Construct a new IRC Server. + * @constructor + * @param {string} domain : The IRC network address + * @param {Object} serverConfig : The config options for this network. + * @param {string} homeserverDomain : The domain of the homeserver + * e.g "matrix.org" + * @param {number} expiryTimeSeconds : How old a matrix message can be + * before it is considered 'expired' and not sent to IRC. If 0, messages + * will never expire. + */ + constructor(public domain: string, public config: IrcServerConfig, + private homeserverDomain: string, private expiryTimeSeconds: number = 0) { + this.addresses = config.additionalAddresses || []; + this.addresses.push(domain); + + if (this.config.dynamicChannels.groupId !== undefined && + this.config.dynamicChannels.groupId.trim() !== "") { + this.groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; + if (!this.groupIdValid) { + log.warn( + `${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` + ); + } + } + else { + this.groupIdValid = false; + } + } + + /** + * Get how old a matrix message can be (in seconds) before it is considered + * 'expired' and not sent to IRC. + * @return {Number} The number of seconds. If 0, they never expire. + */ + public getExpiryTimeSeconds() { + return this.expiryTimeSeconds; + } + + /** + * Get a string that represents the human-readable name for a server. + * @return {string} this.config.name if truthy, otherwise it will return + * an empty string. + */ + public getReadableName() { + return this.config.name || ""; + } + + /** + * Return a randomised server domain from the default and additional addresses. + * @return {string} + */ + public randomDomain() { + return this.addresses[ + Math.floor((Math.random() * 1000) % this.addresses.length) + ]; + } + + /** + * Returns the network ID of this server, which should be unique across all + * IrcServers on the bridge. Defaults to the domain of this IrcServer. + * @return {string} this.config.networkId || this.domain + */ + public getNetworkId() { + return this.config.networkId || this.domain; + } + + /** + * Returns whether the server is configured to wait getQuitDebounceDelayMs before + * parting a user that has disconnected due to a net-split. + * @return {Boolean} this.config.quitDebounce.enabled. + */ + public shouldDebounceQuits() { + return this.config.quitDebounce.enabled; + } + + /** + * Get the minimum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If the user rejoins a channel before bridging + * the quit to a leave, the leave will not be sent. + * @return {number} + */ + public getQuitDebounceDelayMinMs() { + return this.config.quitDebounce.delayMinMs; + } + + /** + * Get the maximum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If a leave is bridged, it will occur at a + * random time between delayMinMs (see above) delayMaxMs. + * @return {number} + */ + public getQuitDebounceDelayMaxMs() { + return this.config.quitDebounce.delayMaxMs; + } + + /** + * Get the rate of maximum quits received per second before a net-split is + * detected. If the rate of quits received becomes higher that this value, + * a net split is considered ongoing. + * @return {number} + */ + public getDebounceQuitsPerSecond() { + return this.config.quitDebounce.quitsPerSecond; + } + + /** + * Get a map that converts IRC user modes to Matrix power levels. + * @return {Object} + */ + public getModePowerMap() { + return this.config.modePowerMap || {}; + } + + public getHardCodedRoomIds() { + const roomIds = new Set(); + const channels = Object.keys(this.config.mappings); + channels.forEach((chan) => { + this.config.mappings[chan].forEach((roomId) => { + roomIds.add(roomId); + }); + }); + return Array.from(roomIds.keys()); + } + + public shouldSendConnectionNotices() { + return this.config.sendConnectionMessages; + } + + public isBotEnabled() { + return this.config.botConfig.enabled; + } + + public getUserModes() { + return this.config.ircClients.userModes || ""; + } + + public getJoinRule() { + return this.config.dynamicChannels.joinRule; + } + + public areGroupsEnabled() { + return this.groupIdValid; + } + + public getGroupId() { + return this.config.dynamicChannels.groupId; + } + + public shouldFederatePMs() { + return this.config.privateMessages.federate; + } + + public getMemberListFloodDelayMs() { + return this.config.membershipLists.floodDelayMs; + } + + public shouldFederate() { + return this.config.dynamicChannels.federate; + } + public forceRoomVersion() { + return this.config.dynamicChannels.roomVersion; + } + + public getPort() { + return this.config.port; + } + + public isInWhitelist(userId: string) { + return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; + } + + public getCA() { + return this.config.ca; + } + + public useSsl() { + return Boolean(this.config.ssl); + } + + public useSslSelfSigned() { + return Boolean(this.config.sslselfsign); + } + + public useSasl() { + return Boolean(this.config.sasl); + } + + public allowExpiredCerts() { + return Boolean(this.config.allowExpiredCerts); + } + + public getIdleTimeout() { + return this.config.ircClients.idleTimeout; + } + + public getReconnectIntervalMs() { + return this.config.ircClients.reconnectIntervalMs; + } + + public getConcurrentReconnectLimit() { + return this.config.ircClients.concurrentReconnectLimit; + } + + public getMaxClients() { + return this.config.ircClients.maxClients; + } + + public shouldPublishRooms() { + return this.config.dynamicChannels.published; + } + + public allowsNickChanges() { + return this.config.ircClients.allowNickChanges; + } + + public getBotNickname() { + return this.config.botConfig.nick; + } + + public createBotIrcClientConfig(username: string) { + return IrcClientConfig.newConfig( + null, this.domain, this.config.botConfig.nick, username, + this.config.botConfig.password + ); + } + + public getIpv6Prefix() { + return this.config.ircClients.ipv6.prefix; + } + + public getIpv6Only() { + return this.config.ircClients.ipv6.only; + } + + public getLineLimit() { + return this.config.ircClients.lineLimit; + } + + public getJoinAttempts() { + return this.config.matrixClients.joinAttempts; + } + + public isExcludedChannel(channel: string) { + return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; + } + + public hasInviteRooms() { + return ( + this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" + ); + } + + // check if this server dynamically create rooms with aliases. + public createsDynamicAliases() { + return ( + this.config.dynamicChannels.enabled && + this.config.dynamicChannels.createAlias + ); + } + + // check if this server dynamically creates rooms which are joinable via an alias only. + public createsPublicAliases() { + return ( + this.createsDynamicAliases() && + this.getJoinRule() === "public" + ); + } + + public allowsPms() { + return this.config.privateMessages.enabled; + } + + public shouldSyncMembershipToIrc(kind: MembershipSyncKind, roomId: string) { + return this._shouldSyncMembership(kind, roomId, true); + } + + public shouldSyncMembershipToMatrix(kind: MembershipSyncKind, channel: string) { + return this._shouldSyncMembership(kind, channel, false); + } + + public _shouldSyncMembership(kind: MembershipSyncKind, identifier: string, toIrc: boolean) { + if (["incremental", "initial"].indexOf(kind) === -1) { + throw new Error("Bad kind: " + kind); + } + if (!this.config.membershipLists.enabled) { + return false; + } + let shouldSync = this.config.membershipLists.global[ + toIrc ? "matrixToIrc" : "ircToMatrix" + ][kind]; + + if (!identifier) { + return shouldSync; + } + + // check for specific rules for the room id / channel + if (toIrc) { + // room rules clobber global rules + this.config.membershipLists.rooms.forEach(function(r) { + if (r.room === identifier && r.matrixToIrc) { + shouldSync = r.matrixToIrc[kind]; + } + }); + } + else { + // channel rules clobber global rules + this.config.membershipLists.channels.forEach(function(chan) { + if (chan.channel === identifier && chan.ircToMatrix) { + shouldSync = chan.ircToMatrix[kind]; + } + }); + } + + return shouldSync; + } + + public shouldJoinChannelsIfNoUsers() { + return this.config.botConfig.joinChannelsIfNoUsers; + } + + public isMembershipListsEnabled() { + return this.config.membershipLists.enabled; + } + + public getUserLocalpart(nick: string) { + // the template is just a literal string with special vars; so find/replace + // the vars and strip the @ + const uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); + return uid.replace(/\$NICK/g, nick).substring(1); + } + + public claimsUserId(userId: string) { + // the server claims the given user ID if the ID matches the user ID template. + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(userId); + } + + public getNickFromUserId(userId: string) { + // extract the nick from the given user ID + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*?)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(userId); + if (!match) { + return null; + } + return match[1]; + } + + public getUserIdFromNick(nick: string) { + const template = this.config.matrixClients.userTemplate; + return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + + ":" + this.homeserverDomain; + } + + public getDisplayNameFromNick(nick: string) { + const template = this.config.matrixClients.displayName; + let displayName = template.replace(/\$NICK/g, nick); + displayName = displayName.replace(/\$SERVER/g, this.domain); + return displayName; + } + + public claimsAlias(alias: string) { + // the server claims the given alias if the alias matches the alias template + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "#(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(alias); + } + + public getChannelFromAlias(alias: string) { + // extract the channel from the given alias + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "([^:]*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(alias); + if (!match) { + return null; + } + log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); + return match[1]; + } + + public getAliasFromChannel(channel: string) { + const template = this.config.dynamicChannels.aliasTemplate; + return template.replace(/\$CHANNEL/, channel) + ":" + this.homeserverDomain; + } + + public getNick(userId: string, displayName?: string) { + const illegalChars = BridgedClient.illegalCharactersRegex; + let localpart = userId.substring(1).split(":")[0]; + localpart = localpart.replace(illegalChars, ""); + displayName = displayName ? displayName.replace(illegalChars, "") : undefined; + const display = [displayName, localpart].find((n) => Boolean(n)); + if (!display) { + throw new Error("Could not get nick for user, all characters were invalid"); + } + const template = this.config.ircClients.nickTemplate; + let nick = template.replace(/\$USERID/g, userId); + nick = nick.replace(/\$LOCALPART/g, localpart); + nick = nick.replace(/\$DISPLAY/g, display); + return nick; + } + + public getAliasRegex() { + return IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public getUserRegex() { + return IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$NICK": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public static get DEFAULT_CONFIG(): IrcServerConfig { + return { + sendConnectionMessages: true, + quitDebounce: { + enabled: false, + quitsPerSecond: 5, + delayMinMs: 3600000, // 1h + delayMaxMs: 7200000, // 2h + }, + botConfig: { + nick: "appservicebot", + joinChannelsIfNoUsers: true, + enabled: true + }, + privateMessages: { + enabled: true, + exclude: [], + federate: true + }, + dynamicChannels: { + enabled: false, + published: true, + createAlias: true, + joinRule: "public", + federate: true, + aliasTemplate: "#irc_$SERVER_$CHANNEL", + whitelist: [], + exclude: [] + }, + mappings: {}, + matrixClients: { + userTemplate: "@$SERVER_$NICK", + displayName: "$NICK (IRC)", + joinAttempts: -1, + }, + ircClients: { + nickTemplate: "M-$DISPLAY", + maxClients: 30, + idleTimeout: 172800, + reconnectIntervalMs: 5000, + concurrentReconnectLimit: 50, + allowNickChanges: false, + ipv6: { + only: false + }, + lineLimit: 3 + }, + membershipLists: { + enabled: false, + floodDelayMs: 10000, // 10s + global: { + ircToMatrix: { + initial: false, + incremental: false + }, + matrixToIrc: { + initial: false, + incremental: false + } + }, + channels: [], + rooms: [] + } + } + } + + private static templateToRegex(template: string, literalVars: {[key: string]: string}, + regexVars: {[key: string]: string}, suffix: string) { + // The 'template' is a literal string with some special variables which need + // to be find/replaced. + let regex = template; + Object.keys(literalVars).forEach(function(varPlaceholder) { + regex = regex.replace( + new RegExp(IrcServer.escapeRegExp(varPlaceholder), 'g'), + literalVars[varPlaceholder] + ); + }); + + // at this point the template is still a literal string, so escape it before + // applying the regex vars. + regex = IrcServer.escapeRegExp(regex); + // apply regex vars + Object.keys(regexVars).forEach(function(varPlaceholder) { + regex = regex.replace( + // double escape, because we bluntly escaped the entire string before + // so our match is now escaped. + new RegExp(IrcServer.escapeRegExp(IrcServer.escapeRegExp(varPlaceholder)), 'g'), + regexVars[varPlaceholder] + ); + }); + + suffix = suffix || ""; + return regex + suffix; + } + + private static escapeRegExp(s: string) { + // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } +} + +export interface IrcServerConfig { + // These are determined to be always defined or possibly undefined + // by the existence of the keys in IrcServer.DEFAULT_CONFIG. + name?: string; + port?: number; + ca?: string; + networkId?: string; + ssl?: boolean; + sslselfsign?: boolean; + sasl?: boolean; + allowExpiredCerts?: boolean; + additionalAddresses?: string[]; + dynamicChannels: { + enabled: boolean; + published: boolean; + createAlias: boolean; + joinRule: "public"|"invite"; + federate: boolean; + aliasTemplate: string; + whitelist: string[]; + exclude: string[]; + roomVersion?: string; + groupId?: string; + }; + quitDebounce: { + enabled: boolean; + quitsPerSecond: number; + delayMinMs: number; + delayMaxMs: number; + }; + mappings: {[channel: string]: string[]}; // chan -> roomId[] + modePowerMap?: {[mode: string]: number}; + sendConnectionMessages: boolean; + botConfig: { + nick: string; + joinChannelsIfNoUsers: boolean; + enabled: boolean; + password?: string; + }; + privateMessages: { + enabled: boolean; + exclude: string[]; + federate: boolean; + }; + matrixClients: { + userTemplate: string; + displayName: string; + joinAttempts: number; + }; + ircClients: { + nickTemplate: string; + maxClients: number; + idleTimeout: number; + reconnectIntervalMs: number; + concurrentReconnectLimit: number; + allowNickChanges: boolean; + ipv6: { + only: boolean; + prefix?: string; + }; + lineLimit: number; + userModes?: string; + }; + membershipLists: { + enabled: boolean; + floodDelayMs: number; + global: { + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }; + channels: { + channel: string; + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + }[]; + rooms: { + room: string; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }[]; + }; +} diff --git a/src/main.js b/src/main.js index 1bec2fa5a..0a5669566 100644 --- a/src/main.js +++ b/src/main.js @@ -7,7 +7,7 @@ const RoomBridgeStore = require("matrix-appservice-bridge").RoomBridgeStore; const UserBridgeStore = require("matrix-appservice-bridge").UserBridgeStore; const IrcBridge = require("./bridge/IrcBridge.js"); -const IrcServer = require("./irc/IrcServer.js"); +const { IrcServer } = require("./irc/IrcServer.js"); const stats = require("./config/stats"); const ident = require("./irc/ident"); const logging = require("./logging"); diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index e90881b37..e51ac7e18 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -17,6 +17,7 @@ limitations under the License. //@ts-ignore import { RemoteRoom } from "matrix-appservice-bridge"; import { toIrcLowerCase } from "../irc/formatting"; +import { IrcServer } from "../irc/IrcServer"; export class IrcRoom extends RemoteRoom { /** @@ -26,7 +27,7 @@ export class IrcRoom extends RemoteRoom { * @param {String} channel : The channel this room represents. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(public readonly server: any, public readonly channel: string) { + constructor(public readonly server: IrcServer, public readonly channel: string) { // Because `super` must be called first, we convert the case several times. super(IrcRoom.createId(server, toIrcLowerCase(channel)), { domain: server.domain, @@ -40,7 +41,7 @@ export class IrcRoom extends RemoteRoom { } getDomain() { - return super.get("domain"); + return super.get("domain") as string; } getServer() { @@ -48,17 +49,17 @@ export class IrcRoom extends RemoteRoom { } getChannel() { - return super.get("channel"); + return super.get("channel") as string; } getType() { - return super.get("type"); + return super.get("type") as string; } // No types for IrcServer yet // eslint-disable-next-line @typescript-eslint/no-explicit-any public static fromRemoteRoom(server: any, remoteRoom: RemoteRoom) { - return new IrcRoom(server, remoteRoom.get("channel")); + return new IrcRoom(server, remoteRoom.get("channel") as string); } // An IRC room is uniquely identified by a combination of the channel name and the From c8a44b0d4e08ee6d57127af2201d000348bb2667 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:38:01 +0100 Subject: [PATCH 014/350] Linting rules --- .eslintrc | 2 +- .ts.eslintrc | 3 ++- package-lock.json | 9 +++++++++ package.json | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.eslintrc b/.eslintrc index 8dd4a6f21..5e754fc44 100644 --- a/.eslintrc +++ b/.eslintrc @@ -59,7 +59,7 @@ "valid-typeof": 2, "array-bracket-spacing": [1, "never"], - "max-len": [1, 100], + "max-len": [1, 120], "brace-style": [1, "stroustrup", { "allowSingleLine": true }], "comma-spacing": [1, {"before": false, "after": true}], "comma-style": [1, "last"], diff --git a/.ts.eslintrc b/.ts.eslintrc index 74fd521d8..d0c9fbcae 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -1,9 +1,10 @@ { "parser": "@typescript-eslint/parser", "plugins": ["@typescript-eslint"], - "extends": ["plugin:@typescript-eslint/recommended"], + "extends": ["plugin:@typescript-eslint/recommended", ".eslintrc"], "rules": { "@typescript-eslint/ban-ts-ignore": 0, "@typescript-eslint/explicit-function-return-type": 0, + "@typescript-eslint/camelcase": ["error", { "properties": "never" }] } } diff --git a/package-lock.json b/package-lock.json index e56545318..4f47f5cee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -164,6 +164,15 @@ "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==" }, + "@types/nedb": { + "version": "1.8.9", + "resolved": "https://registry.npmjs.org/@types/nedb/-/nedb-1.8.9.tgz", + "integrity": "sha512-w9Tl3DQCkdT0Ghes+PKhe+3/pZppBXuFFpSCjPJbb2KE3DjYmUpEyCYzjrAYlT9Y1TndnbbnChzkax2h/JorVQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "10.14.12", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", diff --git a/package.json b/package.json index 927c7eadf..82bff602e 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@typescript-eslint/eslint-plugin": "^2.2.0", "@typescript-eslint/parser": "^2.2.0", "@types/bluebird": "^3.5.27", + "@types/nedb": "^1.8.9", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", From 07c546e2ed9d322b2832360d7e5d6e49902beb58 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:38:11 +0100 Subject: [PATCH 015/350] Create StringCrypto to abstract passwords --- src/datastore/StringCrypto.ts | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/datastore/StringCrypto.ts diff --git a/src/datastore/StringCrypto.ts b/src/datastore/StringCrypto.ts new file mode 100644 index 000000000..b294a786c --- /dev/null +++ b/src/datastore/StringCrypto.ts @@ -0,0 +1,50 @@ +import * as crypto from "crypto"; +import * as fs from "fs"; +import * as logging from "../logging"; + +const log = logging.get("CryptoStore"); + +export class StringCrypto { + private privateKey!: string; + + public load(pkeyPath: string) { + try { + this.privateKey = fs.readFileSync(pkeyPath, "utf8").toString(); + + // Test whether key is a valid PEM key (publicEncrypt does internal validation) + try { + crypto.publicEncrypt( + this.privateKey, + new Buffer("This is a test!") + ); + } + catch (err) { + log.error(`Failed to validate private key: (${err.message})`); + throw err; + } + + log.info(`Private key loaded from ${pkeyPath} - IRC password encryption enabled.`); + } + catch (err) { + log.error(`Could not load private key ${err.message}.`); + throw err; + } + } + + public encrypt(plaintext: string): string { + const salt = crypto.randomBytes(16).toString('base64'); + return crypto.publicEncrypt( + this.privateKey, + new Buffer(salt + ' ' + plaintext) + ).toString('base64'); + } + + public decrypt(encryptedString: string): string { + const decryptedPass = crypto.privateDecrypt( + this.privateKey, + new Buffer(encryptedString, 'base64') + ).toString(); + // Extract the password by removing the prefixed salt and seperating space + return decryptedPass.split(' ')[1]; + } +} From 8afc7d7fcac39b08ce5391db0a67264736d4a1c7 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 16:37:09 +0100 Subject: [PATCH 016/350] Use a type definition file --- src/bridge/IrcBridge.js | 1 - src/datastore/DataStore.ts | 21 ++-- src/datastore/NedbDataStore.ts | 91 +++++++------- src/models/IrcClientConfig.ts | 10 +- tsconfig.json | 5 +- types/matrix-appservice-bridge/index.d.ts | 138 ++++++++++++++++++++++ 6 files changed, 204 insertions(+), 62 deletions(-) create mode 100644 types/matrix-appservice-bridge/index.d.ts diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index edcd08e6b..2c0cdd5d6 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -19,7 +19,6 @@ const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); const { NeDBDataStore } = require("../datastore/NedbDataStore"); -const { PgDataStore } = require("../datastore/postgres/PgDataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index 8f3b5b1c6..e620f4d4b 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -1,20 +1,13 @@ // Ignore definition errors for now. //@ts-ignore -import { MatrixRoom, RemoteRoom, MatrixUser} from "matrix-appservice-bridge"; +import { MatrixRoom, RemoteRoom, MatrixUser, Entry} from "matrix-appservice-bridge"; import {default as Bluebird} from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; export type RoomOrigin = "config"|"provision"|"alias"|"join"; -export interface RoomEntry { - id: string; - matrix: MatrixRoom; - remote: RemoteRoom; - data: { - origin: RoomOrigin; - }; -} + export interface ChannelMappings { [roomId: string]: Array<{networkId: string; channel: string}>; @@ -48,7 +41,7 @@ export interface DataStore { * "join" if it was created during a join. * @return {Promise} A promise which resolves to a room entry, or null if one is not found. */ - getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise; + getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise; /** * Get all Matrix <--> IRC room mappings from the database. @@ -64,7 +57,7 @@ export interface DataStore { * @return {Promise} A promise which resolves to a list * of entries. */ - getProvisionedMappings(roomId: string): Bluebird; + getProvisionedMappings(roomId: string): Bluebird; /** * Remove an IRC <--> Matrix room mapping from the database. @@ -105,9 +98,9 @@ export interface DataStore { getMatrixRoomsForChannel(server: IrcServer, channel: string): Promise>; getMappingsForChannelByOrigin(server: IrcServer, channel: string, - origin: RoomOrigin|RoomOrigin[], allowUnset: boolean): Promise; + origin: RoomOrigin|RoomOrigin[], allowUnset: boolean): Promise; - getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string}>; + getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string[]}>; setModeForRoom(roomId: string, mode: string, enabled: boolean): Promise; @@ -129,7 +122,7 @@ export interface DataStore { storeAdminRoom(room: MatrixRoom, userId: string): Promise; - upsertRoomStoreEntry(entry: RoomEntry): Promise; + upsertRoomStoreEntry(entry: Entry): Promise; getAdminRoomByUserId(userId: string): Promise; diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 42cbed00f..37142b62d 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -16,26 +16,26 @@ limitations under the License. import {default as Bluebird} from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; -import { IrcClientConfig } from "../models/IrcClientConfig" +import { IrcClientConfig, IrcClientConfigSeralized } from "../models/IrcClientConfig" import * as logging from "../logging"; // Ignore definition errors for now. //@ts-ignore -import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom} from "matrix-appservice-bridge"; -import { DataStore, RoomOrigin, ChannelMappings, RoomEntry, UserFeatures } from "./DataStore"; +import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom, UserBridgeStore, RoomBridgeStore, Entry } from "matrix-appservice-bridge"; +import { DataStore, RoomOrigin, ChannelMappings, UserFeatures } from "./DataStore"; import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; import { StringCrypto } from "./StringCrypto"; +import Nedb from "nedb"; const log = logging.get("NeDBDataStore"); + export class NeDBDataStore implements DataStore { private serverMappings: {[domain: string]: IrcServer} = {}; private cryptoStore?: StringCrypto; constructor( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private userStore: any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private roomStore: any, + private userStore: UserBridgeStore, + private roomStore: RoomBridgeStore, pkeyPath: string, private bridgeDomain: string) { const errLog = function(fieldName: string) { @@ -149,15 +149,15 @@ export class NeDBDataStore implements DataStore { * @return {Promise} A promise which resolves to a room entry, or null if one is not found. */ public async getRoom(roomId: string, ircDomain: string, - ircChannel: string, origin?: RoomOrigin): Promise { + ircChannel: string, origin?: RoomOrigin): Promise { if (origin && typeof origin !== "string") { throw new Error(`If defined, origin must be a string = "config"|"provision"|"alias"|"join"`); } const mappingId = NeDBDataStore.createMappingId(roomId, ircDomain, ircChannel); return this.roomStore.getEntryById(mappingId).then( - (entry: RoomEntry) => { - if (origin && entry && origin !== entry.data.origin) { + (entry) => { + if (origin && entry !== null && origin !== entry.data!.origin) { return null; } return entry; @@ -180,17 +180,19 @@ export class NeDBDataStore implements DataStore { const mappings: ChannelMappings = {}; - entries.forEach((e: { matrix_id: string; remote: {domain: string; channel: string}}) => { + entries.forEach((e: Entry) => { + const domain = e.remote!.get("domain") as string; + const channel = e.remote!.get("channel") as string; // drop unknown irc networks in the database - if (!this.serverMappings[e.remote.domain]) { + if (!this.serverMappings[domain]) { return; } if (!mappings[e.matrix_id]) { mappings[e.matrix_id] = []; } mappings[e.matrix_id].push({ - networkId: this.serverMappings[e.remote.domain].getNetworkId(), - channel: e.remote.channel + networkId: this.serverMappings[domain].getNetworkId(), + channel, }); }); @@ -204,9 +206,9 @@ export class NeDBDataStore implements DataStore { * @return {Promise} A promise which resolves to a list * of entries. */ - public getProvisionedMappings(roomId: string): Bluebird { + public getProvisionedMappings(roomId: string): Bluebird { return Bluebird.cast(this.roomStore.getEntriesByMatrixId(roomId)).filter( - (entry: RoomEntry) => { + (entry: Entry) => { return entry.data && entry.data.origin === 'provision' } ); @@ -260,11 +262,12 @@ export class NeDBDataStore implements DataStore { public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{[roomId: string]: IrcRoom[]}> { const roomIdToRemoteRooms: { [roomId: string]: IrcRoom[]; - } = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); - for (const roomId of Object.keys(roomIdToRemoteRooms)) { + } = {}; + const linkedRemoteRooms = await this.roomStore.batchGetLinkedRemoteRooms(roomIds); + for (const roomId of Object.keys(linkedRemoteRooms)) { // filter out rooms with unknown IRC servers and // map RemoteRooms to IrcRooms - roomIdToRemoteRooms[roomId] = roomIdToRemoteRooms[roomId].filter((remoteRoom) => { + roomIdToRemoteRooms[roomId] = linkedRemoteRooms[roomId].filter((remoteRoom) => { return Boolean(this.serverMappings[remoteRoom.get("domain") as string]); }).map((remoteRoom) => { const server = this.serverMappings[remoteRoom.get("domain") as string]; @@ -298,7 +301,7 @@ export class NeDBDataStore implements DataStore { } const remoteId = IrcRoom.createId(server, channel); - return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: RoomEntry[]) => { + return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: Entry[]) => { return entries.filter((e) => { if (allowUnset) { if (!e.data || !e.data.origin) { @@ -310,26 +313,28 @@ export class NeDBDataStore implements DataStore { }); } - public async getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string}> { + public async getModesForChannel (server: IrcServer, channel: string): Promise<{[id: string]: string[]}> { log.info("getModesForChannel (server=%s, channel=%s)", server.domain, channel ); const remoteId = IrcRoom.createId(server, channel); - return this.roomStore.getEntriesByRemoteId(remoteId).then((entries: RoomEntry[]) => { - const mapping: {[id: string]: string[]} = {}; - entries.forEach((entry) => { - mapping[entry.matrix.getId()] = entry.remote.get("modes") as string[] || []; - }); - return mapping; + const entries = await this.roomStore.getEntriesByRemoteId(remoteId); + const mapping: {[id: string]: string[]} = {}; + entries.forEach((entry) => { + mapping[entry.matrix!.getId()] = entry.remote!.get("modes") as string[] || []; }); + return mapping; } public async setModeForRoom(roomId: string, mode: string, enabled = true): Promise { log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", mode, roomId, enabled ); - const entries: RoomEntry[] = await this.roomStore.getEntriesByMatrixId(roomId); + const entries: Entry[] = await this.roomStore.getEntriesByMatrixId(roomId); for (const entry of entries) { + if (!entry.remote) { + return; + } const modes = entry.remote.get("modes") as string[] || []; const hasMode = modes.includes(mode); @@ -371,10 +376,10 @@ export class NeDBDataStore implements DataStore { } public async getTrackedChannelsForServer(domain: string) { - const entries: RoomEntry[] = await this.roomStore.getEntriesByRemoteRoomData({ domain }); + const entries: Entry[] = await this.roomStore.getEntriesByRemoteRoomData({ domain }); const channels: string[] = []; entries.forEach((e) => { - const r = e.remote; + const r = e.remote!; const server = this.serverMappings[r.get("domain") as string]; if (!server) { return; @@ -388,12 +393,12 @@ export class NeDBDataStore implements DataStore { } public async getRoomIdsFromConfig() { - const entries: RoomEntry[] = await this.roomStore.getEntriesByLinkData({ + const entries: Entry[] = await this.roomStore.getEntriesByLinkData({ origin: 'config' }); - return entries.map((e) => { - return e.matrix.getId(); - }); + return entries.map((e) => + e.matrix!.getId() + ); } public async removeConfigMappings() { @@ -412,7 +417,7 @@ export class NeDBDataStore implements DataStore { config.set("ipv6_counter", 0); await this.userStore.setRemoteUser(config); } - return config.get("ipv6_counter"); + return config.get("ipv6_counter") as number; } @@ -431,14 +436,14 @@ export class NeDBDataStore implements DataStore { * @return {Promise} Resolved when the room is retrieved. */ public async getAdminRoomById(roomId: string): Promise { - const entries: RoomEntry[] = await this.roomStore.getEntriesByMatrixId(roomId); + const entries: Entry[] = await this.roomStore.getEntriesByMatrixId(roomId); if (entries.length == 0) { return null; } if (entries.length > 1) { log.error("getAdminRoomById(" + roomId + ") has " + entries.length + " entries"); } - if (entries[0].matrix.get("admin_id")) { + if (entries[0].matrix && entries[0].matrix.get("admin_id")) { return entries[0].matrix; } return null; @@ -455,11 +460,15 @@ export class NeDBDataStore implements DataStore { room.set("admin_id", userId); await this.roomStore.upsertEntry({ id: NeDBDataStore.createAdminId(userId), + matrix_id: room.getId(), matrix: room, + remote: null, + remote_id: "", + data: {}, }); } - public async upsertRoomStoreEntry(entry: RoomEntry): Promise { + public async upsertRoomStoreEntry(entry: Entry): Promise { await this.roomStore.upsertEntry(entry); } @@ -480,7 +489,7 @@ export class NeDBDataStore implements DataStore { if (!matrixUser) { return null; } - const userConfig = matrixUser.get("client_config"); + const userConfig = matrixUser.get("client_config") as any; if (!userConfig) { return null; } @@ -521,7 +530,7 @@ export class NeDBDataStore implements DataStore { if (!user) { user = new MatrixUser(userId); } - const userConfig = user.get("client_config") || {}; + const userConfig = user.get("client_config") as any || {}; const password = config.getPassword(); if (password) { if (!this.cryptoStore) { @@ -540,7 +549,7 @@ export class NeDBDataStore implements DataStore { public async getUserFeatures(userId: string): Promise { const matrixUser = await this.userStore.getMatrixUser(userId); - return matrixUser ? (matrixUser.get("features") || {}) : {}; + return matrixUser ? (matrixUser.get("features") as UserFeatures || {}) : {}; } public async storeUserFeatures(userId: string, features: UserFeatures) { diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index e5b034a0e..46d2b1c59 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -18,7 +18,7 @@ limitations under the License. //@ts-ignore import { MatrixUser } from "matrix-appservice-bridge"; -interface IrcClientConfigSeralized { +export interface IrcClientConfigSeralized { username?: string; password?: string; nick?: string; @@ -38,7 +38,7 @@ export class IrcClientConfig { * @param {Object} configObj Serialised config information if known. */ constructor( - public userId: string, + public userId: string|null, public domain: string, private config: IrcClientConfigSeralized = {}) { @@ -60,7 +60,7 @@ export class IrcClientConfig { return this.config.username; } - public setPassword(password: string) { + public setPassword(password?: string) { this.config.password = password; } @@ -98,8 +98,8 @@ export class IrcClientConfig { return this.userId + "=>" + this.domain + "=" + JSON.stringify(redactedConfig); } - public static newConfig(matrixUser: MatrixUser, domain: string, - nick: string, username: string, password: string) { + public static newConfig(matrixUser: MatrixUser|null, domain: string, + nick: string, username: string, password?: string) { return new IrcClientConfig(matrixUser ? matrixUser.getId() : null, domain, { nick: nick, username: username, diff --git a/tsconfig.json b/tsconfig.json index 4c680298f..a909c0d54 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,10 @@ "outDir": "./lib", "composite": false, "strict": true, - "esModuleInterop": true + "esModuleInterop": true, + "typeRoots": [ + "./types" + ] }, "include": [ "src/**/*" diff --git a/types/matrix-appservice-bridge/index.d.ts b/types/matrix-appservice-bridge/index.d.ts new file mode 100644 index 000000000..e86926dc9 --- /dev/null +++ b/types/matrix-appservice-bridge/index.d.ts @@ -0,0 +1,138 @@ +/* +Copyright 2019 Huan LI +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * This has been borrowed from https://github.com/huan/matrix-appservice-wechaty/blob/master/src/typings/matrix-appservice-bridge.d.ts + * under the Apache2 licence. + */ +declare module 'matrix-appservice-bridge' { + interface RoomMemberDict { + [id: string]: { + display_name: string, + avatar_url: string, + } + } + interface RemoteRoomDict { + [id: string]: RemoteRoom[] + } + interface EntryDict { + [id: string]: Array + } + + export interface Entry { + id: string // The unique ID for this entry. + matrix_id: string // "room_id", + remote_id: string // "remote_room_id", + matrix: null|MatrixRoom // The matrix room, if applicable. + remote: null|RemoteRoom // The remote room, if applicable. + data: null|any // Information about this mapping, which may be an empty. + } + + export class MatrixRoom { + protected roomId: string + + constructor (roomId: string, data?: object) + deserialize(data: object): void + get(key: string): unknown + getId(): string + serialize(): object + set(key: string, val: any): void + } + + export class MatrixUser { + public readonly localpart: string + public readonly host: string + + private userId: string + + constructor (userId: string, data?: object, escape?: boolean) + escapeUserId(): void + get(key: string): unknown + getDisplayName(): null|string + getId(): string + serialize(): object + set(key: string, val: any): void + setDisplayName(name: string): void + } + + export class RemoteRoom { + constructor (identifier: string, data?: object) + get(key: string): unknown + getId(): string + serialize(): object + set(key: string, val: object|string|number): void + } + + export class RemoteUser { + constructor (id: string, data?: object) + get(key: string): unknown + getId(): string + serialize(): object + set(key: string, val: object|string|number): void + } + + export class BridgeStore { + db: Nedb + delete (query: any): Promise + insert (query: any): Promise + select (query: any, transformFn?: (item: Entry) => Entry): Promise + } + + export class RoomBridgeStore extends BridgeStore { + batchGetLinkedRemoteRooms (matrixIds: Array): Promise + getEntriesByLinkData (data: object): Promise> + getEntriesByMatrixId (matrixId: string): Promise> + getEntriesByMatrixIds (ids: Array): Promise + getEntriesByMatrixRoomData (data: object): Promise> + getEntriesByRemoteId (remoteId: string): Promise> + getEntriesByRemoteRoomData (data: object): Promise> + getEntryById (id: string): Promise + getLinkedMatrixRooms (remoteId: string): Promise> + getLinkedRemoteRooms (matrixId: string): Promise> + getMatrixRoom (roomId: string): Promise + removeEntriesByLinkData (data: object): Promise + removeEntriesByMatrixRoomData (data: object): Promise + removeEntriesByMatrixRoomId (matrixId: string): Promise + removeEntriesByRemoteRoomData (data: object): Promise + removeEntriesByRemoteRoomId (remoteId: string): Promise + setMatrixRoom (matrixRoom: MatrixRoom): Promise + upsertEntry (entry: Entry): Promise + linkRooms ( + matrixRoom: MatrixRoom, + remoteRoom: RemoteRoom, + data?: object, + linkId?: string, + ): Promise + } + + export class UserBridgeStore extends BridgeStore { + getByMatrixData (dataQuery: object): Promise> + getByMatrixLocalpart (localpart: string): Promise + getByRemoteData (dataQuery: object): Promise> + getMatrixLinks (remoteId: string): Promise> + getMatrixUser (userId: string): Promise + getMatrixUsersFromRemoteId (remoteId: string): Promise> + getRemoteLinks (matrixId: string): Promise> + getRemoteUser (id: string): Promise + getRemoteUsersFromMatrixId (userId: string): Promise> + linkUsers (matrixUser: MatrixUser, remoteUser: RemoteUser): Promise + setMatrixUser (matrixUser: MatrixUser): Promise + setRemoteUser (remoteUser: RemoteUser): Promise + unlinkUserIds (matrixUserId: string, remoteUserId: string): Promise + unlinkUsers (matrixUser: MatrixUser, remoteUser: RemoteUser): Promise + } +} \ No newline at end of file From 76dc811c6a3b3e396e90b3f8a1620ae9a53b44aa Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 16:42:58 +0100 Subject: [PATCH 017/350] This isn't an entry --- src/datastore/NedbDataStore.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 37142b62d..aee4813b8 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -180,9 +180,9 @@ export class NeDBDataStore implements DataStore { const mappings: ChannelMappings = {}; - entries.forEach((e: Entry) => { - const domain = e.remote!.get("domain") as string; - const channel = e.remote!.get("channel") as string; + entries.forEach((e: any) => { + const domain = e.remote.domain; + const channel = e.remote.channel; // drop unknown irc networks in the database if (!this.serverMappings[domain]) { return; From 35056cc0caa19c9b94ca9b6719b5123668807e79 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 16:53:20 +0100 Subject: [PATCH 018/350] Linting --- src/datastore/DataStore.ts | 4 +-- src/datastore/NedbDataStore.ts | 39 ++++++++++++++--------- types/matrix-appservice-bridge/index.d.ts | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index e620f4d4b..b051957f1 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -1,6 +1,4 @@ -// Ignore definition errors for now. -//@ts-ignore -import { MatrixRoom, RemoteRoom, MatrixUser, Entry} from "matrix-appservice-bridge"; +import { MatrixRoom, MatrixUser, Entry} from "matrix-appservice-bridge"; import {default as Bluebird} from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig } from "../models/IrcClientConfig"; diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index aee4813b8..ab939db6f 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -19,16 +19,17 @@ import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig, IrcClientConfigSeralized } from "../models/IrcClientConfig" import * as logging from "../logging"; -// Ignore definition errors for now. -//@ts-ignore -import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom, UserBridgeStore, RoomBridgeStore, Entry } from "matrix-appservice-bridge"; +import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom, + UserBridgeStore, RoomBridgeStore, Entry } from "matrix-appservice-bridge"; import { DataStore, RoomOrigin, ChannelMappings, UserFeatures } from "./DataStore"; import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; import { StringCrypto } from "./StringCrypto"; -import Nedb from "nedb"; const log = logging.get("NeDBDataStore"); +interface ClientConfigMap { + [domain: string]: IrcClientConfigSeralized; +} export class NeDBDataStore implements DataStore { private serverMappings: {[domain: string]: IrcServer} = {}; @@ -157,7 +158,7 @@ export class NeDBDataStore implements DataStore { const mappingId = NeDBDataStore.createMappingId(roomId, ircDomain, ircChannel); return this.roomStore.getEntryById(mappingId).then( (entry) => { - if (origin && entry !== null && origin !== entry.data!.origin) { + if (origin && entry && entry.data && origin !== entry.data.origin) { return null; } return entry; @@ -180,7 +181,7 @@ export class NeDBDataStore implements DataStore { const mappings: ChannelMappings = {}; - entries.forEach((e: any) => { + entries.forEach((e: { remote: { domain: string; channel: string}; matrix_id: string}) => { const domain = e.remote.domain; const channel = e.remote.channel; // drop unknown irc networks in the database @@ -321,7 +322,8 @@ export class NeDBDataStore implements DataStore { const entries = await this.roomStore.getEntriesByRemoteId(remoteId); const mapping: {[id: string]: string[]} = {}; entries.forEach((entry) => { - mapping[entry.matrix!.getId()] = entry.remote!.get("modes") as string[] || []; + if (!entry.matrix || !entry.remote) { return; } + mapping[entry.matrix.getId()] = entry.remote.get("modes") as string[] || []; }); return mapping; } @@ -379,12 +381,14 @@ export class NeDBDataStore implements DataStore { const entries: Entry[] = await this.roomStore.getEntriesByRemoteRoomData({ domain }); const channels: string[] = []; entries.forEach((e) => { - const r = e.remote!; - const server = this.serverMappings[r.get("domain") as string]; + if (!e.remote) { + return; + } + const server = this.serverMappings[e.remote.get("domain") as string]; if (!server) { return; } - const ircRoom = IrcRoom.fromRemoteRoom(server, r); + const ircRoom = IrcRoom.fromRemoteRoom(server, e.remote); if (ircRoom.getType() === "channel") { channels.push(ircRoom.getChannel()); } @@ -396,9 +400,12 @@ export class NeDBDataStore implements DataStore { const entries: Entry[] = await this.roomStore.getEntriesByLinkData({ origin: 'config' }); - return entries.map((e) => - e.matrix!.getId() - ); + return entries.map((e) => { + if (!e.matrix) { + return ""; + } + return e.matrix.getId(); + }).filter((e) => e !== ""); } public async removeConfigMappings() { @@ -489,7 +496,8 @@ export class NeDBDataStore implements DataStore { if (!matrixUser) { return null; } - const userConfig = matrixUser.get("client_config") as any; + + const userConfig = matrixUser.get("client_config") as ClientConfigMap; if (!userConfig) { return null; } @@ -530,7 +538,8 @@ export class NeDBDataStore implements DataStore { if (!user) { user = new MatrixUser(userId); } - const userConfig = user.get("client_config") as any || {}; + + const userConfig = user.get("client_config") as ClientConfigMap || {}; const password = config.getPassword(); if (password) { if (!this.cryptoStore) { diff --git a/types/matrix-appservice-bridge/index.d.ts b/types/matrix-appservice-bridge/index.d.ts index e86926dc9..a4401628d 100644 --- a/types/matrix-appservice-bridge/index.d.ts +++ b/types/matrix-appservice-bridge/index.d.ts @@ -89,7 +89,7 @@ declare module 'matrix-appservice-bridge' { db: Nedb delete (query: any): Promise insert (query: any): Promise - select (query: any, transformFn?: (item: Entry) => Entry): Promise + select (query: any, transformFn?: (item: Entry) => Entry): Promise } export class RoomBridgeStore extends BridgeStore { From cc98e1ba904838407e1cef6ffd6c091ded521a5a Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:40:54 +0100 Subject: [PATCH 019/350] Add PgDataStore --- package-lock.json | 117 +++++- package.json | 2 + src/bridge/IrcBridge.js | 10 + src/datastore/postgres/PgDataStore.ts | 489 ++++++++++++++++++++++++++ src/datastore/postgres/schema/v1.ts | 75 ++++ src/main.js | 12 +- src/models/IrcClientConfig.ts | 9 +- 7 files changed, 706 insertions(+), 8 deletions(-) create mode 100644 src/datastore/postgres/PgDataStore.ts create mode 100644 src/datastore/postgres/schema/v1.ts diff --git a/package-lock.json b/package-lock.json index 4f47f5cee..e959fc375 100644 --- a/package-lock.json +++ b/package-lock.json @@ -178,6 +178,23 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" }, + "@types/pg": { + "version": "7.11.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.11.1.tgz", + "integrity": "sha512-ayO8XV0xuJV3cEY4wySyD/7MA1HL75UpvJ5JAme00kNWA5pddlGtN4BRG97xgGe2NHgwxN8AkdjTQUEDypM8Uw==", + "requires": { + "@types/node": "*", + "@types/pg-types": "*" + } + }, + "@types/pg-types": { + "version": "1.11.4", + "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.4.tgz", + "integrity": "sha512-WdIiQmE347LGc1Vq3Ki8sk3iyCuLgnccqVzgxek6gEHp2H0p3MQ3jniIHt+bRODXKju4kNQ+mp53lmP5+/9moQ==", + "requires": { + "moment": ">=2.14.0" + } + }, "@typescript-eslint/eslint-plugin": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.2.0.tgz", @@ -531,6 +548,11 @@ "ieee754": "^1.1.4" } }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "builtin-modules": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", @@ -2916,6 +2938,11 @@ "release-zalgo": "^1.0.0" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -2990,6 +3017,62 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "pg": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.12.1.tgz", + "integrity": "sha512-l1UuyfEvoswYfcUe6k+JaxiN+5vkOgYcVSbSuw3FvdLqDbaoa2RJo1zfJKfPsSYPFVERd4GHvX3s2PjG1asSDA==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-pool": "^2.0.4", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.7.tgz", + "integrity": "sha512-UiJyO5B9zZpu32GSlP0tXy8J2NsJ9EFGFfz5v6PSbdz/1hBLX1rNiiy5+mAm5iJJYwfCv4A0EBcQLGWwjbpzZw==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", @@ -3024,6 +3107,29 @@ } } }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", + "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -3476,6 +3582,14 @@ "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", "dev": true }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3653,8 +3767,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "tmp": { "version": "0.0.33", diff --git a/package.json b/package.json index 82bff602e..95c33d121 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "matrix-lastactive": "^0.0.8", "nedb": "^1.1.2", "nopt": "^3.0.1", + "pg": "^7.12.1", "prom-client": "^6.3.0", "request": "^2.54.0", "sanitize-html": "^1.6.1", @@ -50,6 +51,7 @@ "@typescript-eslint/parser": "^2.2.0", "@types/bluebird": "^3.5.27", "@types/nedb": "^1.8.9", + "@types/pg": "^7.11.1", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 2c0cdd5d6..578550f23 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -318,12 +318,22 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; + const dbConfig = this.config.ircService.database; + if (dbConfig && dbConfig.engine === "postgres") { + this._dataStore = new PgDataStore(this.config.homeserver.domain, dbConfig.connectionString, pkeyPath); + yield this._dataStore.ensureSchema(); + } + else if (dbConfig) { + throw Error("Incorrect database config"); + } + else { this._dataStore = new NeDBDataStore( this._bridge.getUserStore(), this._bridge.getRoomStore(), pkeyPath, this.config.homeserver.domain, ); + } yield this._dataStore.removeConfigMappings(); this._identGenerator = new IdentGenerator(this._dataStore); diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts new file mode 100644 index 000000000..63916a9ba --- /dev/null +++ b/src/datastore/postgres/PgDataStore.ts @@ -0,0 +1,489 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Pool } from "pg"; + +// eslint-disable-next-line @typescript-eslint/no-duplicate-imports + +import { MatrixUser, MatrixRoom, RemoteRoom } from "matrix-appservice-bridge"; +import { DataStore, RoomOrigin, ChannelMappings, RoomEntry, UserFeatures } from "../DataStore"; +import { IrcRoom } from "../../models/IrcRoom"; +import { IrcClientConfig } from "../../models/IrcClientConfig"; +import { IrcServer, IrcServerConfig } from "../../irc/IrcServer"; + +import * as logging from "../../logging"; +import Bluebird from "bluebird"; +import { stat } from "fs"; +import { StringCrypto } from "../StringCrypto"; + +const log = logging.get("PgDatastore"); + +export class PgDataStore implements DataStore { + private serverMappings: {[domain: string]: IrcServer} = {}; + + public static readonly LATEST_SCHEMA = 1; + private pgPool: Pool; + private cryptoStore?: StringCrypto; + + constructor(private bridgeDomain: string, connectionString: string, pkeyPath?: string, min: number = 1, max: number = 4) { + this.pgPool = new Pool({ + connectionString, + min, + max, + }); + if (pkeyPath) { + this.cryptoStore = new StringCrypto(); + this.cryptoStore.load(pkeyPath); + } + process.on("beforeExit", (e) => { + // Ensure we clean up on exit + this.pgPool.end(); + }) + } + + public async setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise { + this.serverMappings[server.domain] = server; + + for (const channel of Object.keys(serverConfig.mappings)) { + const ircRoom = new IrcRoom(server, channel); + for (const roomId of serverConfig.mappings[channel]) { + const mxRoom = new MatrixRoom(roomId); + await this.storeRoom(ircRoom, mxRoom, "config"); + } + } + } + + public async storeRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, origin: RoomOrigin): Promise { + if (typeof origin !== "string") { + throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); + } + log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", + matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin); + this.upsertRoom( + origin, + ircRoom.getType(), + ircRoom.getDomain(), + ircRoom.getChannel(), + matrixRoom.getId(), + JSON.stringify(ircRoom.serialize()), + JSON.stringify(matrixRoom.serialize()), + ); + } + + public async upsertRoom(origin: RoomOrigin, type: string, domain: string, channel: string, roomId: string, ircJson: string, matrixJson: string) { + const parameters = { + origin, + type, + irc_domain: domain, + irc_channel: channel, + room_id: roomId, + irc_json: ircJson, + matrix_json: matrixJson, + }; + const statement = PgDataStore.BuildUpsertStatement("rooms","ON CONSTRAINT cons_rooms_unique", Object.keys(parameters)); + await this.pgPool.query(statement, Object.values(parameters)); + } + + private static pgToRoomEntry(pgEntry: any): RoomEntry { + return { + id: "", + matrix: new MatrixRoom(pgEntry.room_id, JSON.parse(pgEntry.matrix_json)), + remote: new RemoteRoom("", JSON.parse(pgEntry.irc_json)), + data: { + origin: pgEntry.origin, + }, + }; + } + + public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise { + let statement = "SELECT * FROM rooms WHERE room_id = $1, irc_domain = $2, irc_channel = $3"; + if (origin) { + statement += ", origin = $4"; + } + const pgEntry = await this.pgPool.query(statement, [roomId, ircDomain, ircChannel, origin]); + if (!pgEntry.rowCount) { + return null; + } + return PgDataStore.pgToRoomEntry(pgEntry.rows[0]); + } + + public async getAllChannelMappings(): Promise { + const entries = (await this.pgPool.query("SELECT irc_domain, room_id, irc_channel FROM rooms WHERE type = 'channel'")).rows; + + const mappings: ChannelMappings = {}; + const validDomains = Object.keys(this.serverMappings); + entries.forEach((e) => { + if (!e.room_id) { + return; + } + // Filter out servers we don't know about + if (!validDomains.includes(e.irc_domain)) { + return; + } + if (!mappings[e.room_id]) { + mappings[e.room_id] = []; + } + mappings[e.room_id].push({ + networkId: this.serverMappings[e.irc_domain].getNetworkId(), + channel: e.irc_channel, + }); + }) + + return mappings; + } + + public getEntriesByMatrixId(roomId: string): Bluebird { + return Bluebird.cast(this.pgPool.query("SELECT * FROM rooms WHERE room_id = $1", [ + roomId + ])).then((result) => result.rows).map((e) => PgDataStore.pgToRoomEntry(e)); + } + + public getProvisionedMappings(roomId: string): Bluebird { + return Bluebird.cast(this.pgPool.query("SELECT * FROM rooms WHERE room_id = $1 AND origin = 'provision'", [ + roomId + ])).then((result) => result.rows).map((e) => PgDataStore.pgToRoomEntry(e)); + } + + public async removeRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise { + await this.pgPool.query( + "DELETE FROM rooms WHERE room_id = $1, irc_domain = $2, irc_channel = $3, origin = $4", + [roomId, ircDomain, ircChannel, origin] + ); + } + + public async getIrcChannelsForRoomId(roomId: string): Promise { + const entries = await this.pgPool.query("SELECT irc_domain, irc_channel FROM rooms WHERE room_id = $1", [ roomId ]); + return entries.rows.map((e) => { + const server = this.serverMappings[e.irc_domain]; + if (!server) { + // ! is used here because typescript doesn't understand the .filter + return undefined!; + } + return new IrcRoom(server, e.irc_channel); + }).filter((i) => i !== undefined); + } + + public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{ [roomId: string]: IrcRoom[]; }> { + const entries = await this.pgPool.query("SELECT room_id, irc_domain, irc_channel FROM rooms WHERE room_id IN $1", [ + roomIds + ]); + const mapping: { [roomId: string]: IrcRoom[]; } = {}; + entries.rows.forEach((e) => { + const server = this.serverMappings[e.irc_domain]; + if (!server) { + // ! is used here because typescript doesn't understand the .filter + return; + } + if (!mapping[e.room_id]) { + mapping[e.room_id] = []; + } + mapping[e.room_id].push(new IrcRoom(server, e.irc_channel)); + }); + return mapping; + } + + public async getMatrixRoomsForChannel(server: IrcServer, channel: string): Promise { + const entries = await this.pgPool.query("SELECT room_id, matrix_json FROM rooms WHERE irc_domain = $1 AND irc_channel = $2", + [ + server.domain, + channel, + ]); + return entries.rows.map((e) => new MatrixRoom(e.room_id, JSON.parse(e.matrix_json))); + } + + public async getMappingsForChannelByOrigin(server: IrcServer, channel: string, origin: "config" | "provision" | "alias" | "join" | RoomOrigin[], allowUnset: boolean): Promise { + const entries = await this.pgPool.query("SELECT * FROM rooms WHERE irc_domain = $1 AND irc_channel = $2 AND origin = $3", + [ + server.domain, + channel, + origin, + ]); + return entries.rows.map((e) => PgDataStore.pgToRoomEntry(e)); + } + + public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string; }> { + const mapping: {[id: string]: string} = {}; + const entries = await this.pgPool.query( + "SELECT room_id, remote_json->>'modes' AS MODES FROM rooms " + + "WHERE irc_domain = $1 AND irc_channel = $2", + [ + server.domain, + channel, + ]); + entries.rows.forEach((e) => { + mapping[e.room_id] = e.modes; + }); + return mapping; + } + + public async setModeForRoom(roomId: string, mode: string, enabled: boolean): Promise { + log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", + mode, roomId, enabled + ); + const entries: RoomEntry[] = await this.getEntriesByMatrixId(roomId); + for (const entry of entries) { + const modes = entry.remote.get("modes") as string[] || []; + const hasMode = modes.includes(mode); + + if (hasMode === enabled) { + continue; + } + if (enabled) { + modes.push(mode); + } + else { + modes.splice(modes.indexOf(mode), 1); + } + + entry.remote.set("modes", modes); + await this.pgPool.query("UPDATE rooms WHERE room_id = $1, irc_channel = $2, irc_domain = $3 SET irc_json = $4", [ + roomId, + entry.remote.get("channel"), + entry.remote.get("domain"), + JSON.stringify(entry.remote.serialize()), + ]); + } + } + + public async setPmRoom(ircRoom: IrcRoom, matrixRoom: MatrixRoom, userId: string, virtualUserId: string): Promise { + await this.pgPool.query("INSERT INTO pm_rooms VALUES ($1, $2, $3, $4, $5)", [ + matrixRoom.getId(), + ircRoom.getDomain(), + ircRoom.getChannel(), + userId, + virtualUserId, + ]); + } + + public async getMatrixPmRoom(realUserId: string, virtualUserId: string): Promise { + const res = await this.pgPool.query("SELECT room_id FROM pm_rooms WHERE matrix_user_id = $1 AND virtual_user_id = $2", [ + realUserId, + virtualUserId, + ]); + if (res.rowCount === 0) { + return null; + } + return new MatrixRoom(res.rows[0].room_id); + } + + public async getTrackedChannelsForServer(domain: string): Promise { + if (this.serverMappings[domain]) { + return []; + } + const chanSet = await this.pgPool.query("SELECT channel FROM rooms WHERE irc_domain = $1", [ domain ]); + return [...new Set((chanSet.rows).map((e) => e.channel))]; + } + + public async getRoomIdsFromConfig(): Promise { + return ( + await this.pgPool.query("SELECT room_id FROM rooms WHERE origin = 'config'") + ).rows.map((e) => e.room_id); + } + + public async removeConfigMappings(): Promise { + await this.pgPool.query("DELETE FROM rooms WHERE origin = 'config'"); + } + + public async getIpv6Counter(): Promise { + const res = await this.pgPool.query("SELECT counter FROM ipv6_counter"); + return res ? res.rows[0].counter : 0; + } + + public async setIpv6Counter(counter: number): Promise { + await this.pgPool.query("UPDATE ipv6_counter SET count = $1", [ counter ]); + } + + public async upsertRoomStoreEntry(entry: RoomEntry): Promise { + throw new Error("Method not implemented."); + } + + public async getAdminRoomById(roomId: string): Promise { + const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE room_id = $1", [ roomId ]); + if (res.rowCount === 0) { + return null; + } + return new MatrixRoom(roomId); + } + + public async storeAdminRoom(room: MatrixRoom, userId: string): Promise { + await this.pgPool.query(PgDataStore.BuildUpsertStatement("admin_rooms", "(room_id)", [ + "room_id", + "user_id", + ]), [ room.getId(), userId ]); + } + + public async getAdminRoomByUserId(userId: string): Promise { + const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE user_id = $1", [ userId ]); + if (res.rowCount === 0) { + return null; + } + return new MatrixRoom(res.rows[0].room_id); + } + + public async storeMatrixUser(matrixUser: MatrixUser): Promise { + const parameters = { + user_id: matrixUser.getId(), + data: JSON.stringify(matrixUser.serialize()), + }; + const statement = PgDataStore.BuildUpsertStatement("matrix_users", "(user_id)", Object.keys(parameters)); + await this.pgPool.query(statement, Object.values(parameters)); + } + + public async getIrcClientConfig(userId: string, domain: string): Promise { + const res = await this.pgPool.query("SELECT config, password FROM client_config WHERE user_id = $1 and domain = $2", + [ + userId, + domain + ]); + if (res.rowCount === 0) { + return null; + } + const row = res.rows[0]; + let config = JSON.parse(row.config); + if (row.password && this.cryptoStore) { + config.password = this.cryptoStore.decrypt(row.password); + } + return new IrcClientConfig(userId, domain, config); + } + + public async storeIrcClientConfig(config: IrcClientConfig): Promise { + const userId = config.getUserId(); + if (!userId) { + throw Error("IrcClientConfig does not contain a userId"); + } + let password = undefined; + if (config.getPassword() && this.cryptoStore) { + password = this.cryptoStore.encrypt(config.getPassword()!); + } + const parameters = { + user_id: userId, + domain: config.getDomain(), + // either use the decrypted password, or whatever is stored already. + password: password || config.getPassword()!, + config: JSON.stringify(config.serialize(true)), + }; + const statement = PgDataStore.BuildUpsertStatement("client_config", "ON CONSTRAINT cons_client_config_unique", Object.keys(parameters)); + await this.pgPool.query(statement, Object.values(parameters)); + } + + public async getMatrixUserByLocalpart(localpart: string): Promise { + const res = await this.pgPool.query("SELECT user_id, data FROM matrix_users WHERE user_id = $1", [ + `@${localpart}:${this.bridgeDomain}`, + ]); + if (res.rowCount === 0) { + return null; + } + const row = res.rows[0]; + return new MatrixUser(row.user_id, JSON.parse(row.data)); + } + + public async getUserFeatures(userId: string): Promise { + const pgRes = ( + await this.pgPool.query("SELECT features FROM user_features WHERE user_id = $1", + [ userId ]) + ); + if (pgRes.rowCount === 0) { + return {}; + } + return pgRes.rows[0].features; + } + + public async storeUserFeatures(userId: string, features: UserFeatures): Promise { + const statement = PgDataStore.BuildUpsertStatement("user_features", "(user_id)", [ + "user_id", + "features", + ]); + await this.pgPool.query(statement, [userId, JSON.stringify(features)]); + } + + public async storePass(userId: string, domain: string, pass: string, encrypt: boolean = true): Promise { + let password = pass; + if (encrypt) { + if (!this.cryptoStore) { + throw Error("Password encryption is not configured.") + } + password = this.cryptoStore.encrypt(pass); + } + const parameters = { + user_id: userId, + domain, + password, + }; + const statement = PgDataStore.BuildUpsertStatement("client_config", "ON CONSTRAINT cons_client_config_unique", Object.keys(parameters)); + await this.pgPool.query(statement, Object.values(parameters)); + } + + public async removePass(userId: string, domain: string): Promise { + await this.pgPool.query("DELETE FROM user_password WHERE user_id = ${user_id} AND domain = ${domain}"); + } + + public async getMatrixUserByUsername(domain: string, username: string): Promise { + // This will need a join + const res = await this.pgPool.query("SELECT client_config.user_id, matrix_users.data FROM client_config, matrix_users" + + "WHERE config->>'username' = $1 AND domain = $2 AND client_config.user_id = matrix_users.user_id", + [username, domain] + ); + if (res.rowCount === 0) { + return; + } + return new MatrixUser(res.rows[0].user_id, JSON.parse(res.rows[0].data)); + } + + public async ensureSchema() { + log.info("Starting postgres database engine"); + let currentVersion = await this.getSchemaVersion(); + while (currentVersion < PgDataStore.LATEST_SCHEMA) { + log.info(`Updating schema to v${currentVersion + 1}`); + const runSchema = require(`./schema/v${currentVersion + 1}`).runSchema; + try { + await runSchema(this.pgPool); + currentVersion++; + await this.updateSchemaVersion(currentVersion); + } catch (ex) { + log.warn(`Failed to run schema v${currentVersion + 1}:`, ex); + throw Error("Failed to update database schema"); + } + } + log.info(`Database schema is at version v${currentVersion}`); + } + + private async updateSchemaVersion(version: number) { + log.debug(`updateSchemaVersion: ${version}`); + await this.pgPool.query("UPDATE schema SET version = $1;", [ version ]); + } + + private async getSchemaVersion(): Promise { + try { + const { rows } = await this.pgPool.query("SELECT version FROM SCHEMA"); + return rows[0].version; + } catch (ex) { + if (ex.code === "42P01") { // undefined_table + log.warn("Schema table could not be found"); + return 0; + } + log.error("Failed to get schema version:", ex); + } + throw Error("Couldn't fetch schema version"); + } + + private static BuildUpsertStatement(table: string, constraint: string, keyNames: string[]): string { + const keys = keyNames.join(", "); + const keysValues = `\$${keyNames.map((k, i) => i + 1).join(", $")}`; + const keysSets = keyNames.map((k, i) => `${k} = \$${i + 1}`).join(", "); + const statement = `INSERT INTO ${table} (${keys}) VALUES (${keysValues}) ON CONFLICT ${constraint} DO UPDATE SET ${keysSets}`; + return statement; + } +} \ No newline at end of file diff --git a/src/datastore/postgres/schema/v1.ts b/src/datastore/postgres/schema/v1.ts new file mode 100644 index 000000000..d58b02419 --- /dev/null +++ b/src/datastore/postgres/schema/v1.ts @@ -0,0 +1,75 @@ +import { PoolClient } from "pg"; + +// tslint:disable-next-line: no-any +export async function runSchema(connection: PoolClient) { + // Create schema + await connection.query(` + CREATE TABLE schema ( + version INTEGER UNIQUE NOT NULL + ); + + INSERT INTO schema VALUES (0); + + CREATE TABLE rooms ( + origin TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + irc_domain TEXT NOT NULL, + irc_channel TEXT NOT NULL, + irc_json JSON NOT NULL, + matrix_json JSON NOT NULL, + CONSTRAINT cons_rooms_unique UNIQUE(room_id, irc_domain, irc_channel) + ); + + CREATE INDEX rooms_roomid_idx ON rooms (room_id); + CREATE INDEX rooms_ircdomainchannel_idx ON rooms (irc_domain, irc_channel); + + CREATE TABLE admin_rooms ( + room_id TEXT UNIQUE, + user_id TEXT + ); + + CREATE TABLE pm_rooms ( + room_id TEXT UNIQUE, + irc_domain TEXT NOT NULL, + irc_nick TEXT NOT NULL, + matrix_user_id TEXT, + virtual_user_id TEXT, + CONSTRAINT cons_pm_rooms_matrix_irc_unique UNIQUE(matrix_user_id, irc_domain, irc_nick) + ); + + CREATE TABLE matrix_users ( + user_id TEXT UNIQUE, + data TEXT + ); + + CREATE TABLE client_config ( + user_id TEXT, + domain TEXT NOT NULL, + config JSON, + password TEXT, + CONSTRAINT cons_client_config_unique UNIQUE(user_id, domain) + ); + + CREATE TABLE user_features ( + user_id TEXT UNIQUE, + features JSON + ); + + CREATE TABLE ipv6_counter ( + count INTEGER + ); + + CREATE TABLE ( + origin TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + irc_domain TEXT NOT NULL, + irc_channel TEXT NOT NULL, + irc_json JSON NOT NULL, + matrix_json JSON NOT NULL, + CONSTRAINT cons_rooms_unique UNIQUE(room_id, irc_domain, irc_channel) + ); + + INSERT INTO ipv6_counter VALUES (0);`); +} diff --git a/src/main.js b/src/main.js index 0a5669566..40ec5a339 100644 --- a/src/main.js +++ b/src/main.js @@ -114,10 +114,6 @@ module.exports.runBridge = Promise.coroutine(function*(port, config, reg, isDBIn require("http").globalAgent.maxSockets = maxSockets; require("https").globalAgent.maxSockets = maxSockets; - // backwards compat for 1 release. TODO remove - if (config.appService && !config.homeserver) { - config.homeserver = config.appService.homeserver; - } // run the bridge const ircBridge = new IrcBridge(config, reg); @@ -126,6 +122,14 @@ module.exports.runBridge = Promise.coroutine(function*(port, config, reg, isDBIn ircBridge._bridge.opts.roomStore = new RoomBridgeStore(new Datastore()); ircBridge._bridge.opts.userStore = new UserBridgeStore(new Datastore()); } + else if (config.database && config.database.engine === "postgres") { + // Enforce these not to be created + ircBridge._bridge.opts.roomStore = undefined; + ircBridge._bridge.opts.userStore = undefined; + } + else if (config.database) { + throw Error("Invalid database configuration"); + } yield ircBridge.run(port); return ircBridge; diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index 46d2b1c59..bc2883942 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -48,7 +48,7 @@ export class IrcClientConfig { return this.domain; } - public getUserId() { + public getUserId(): string|null { return this.userId; } @@ -84,7 +84,12 @@ export class IrcClientConfig { return this.config.ipv6; } - public serialize() { + public serialize(removePassword = false) { + if (removePassword) { + const clone = JSON.parse(JSON.stringify(this.config)); + delete clone.password; + return clone; + } return this.config; } From bb393aaaf9fbf698f280fbfd1a5fdf081d2f8a32 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 17:11:55 +0100 Subject: [PATCH 020/350] Keep changes on parent branch aligned --- package-lock.json | 2 ++ src/bridge/IrcBridge.js | 1 + src/datastore/NedbDataStore.ts | 2 +- src/datastore/postgres/PgDataStore.ts | 27 ++++++++++++++++----------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index e959fc375..b6bdf8fc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -182,6 +182,7 @@ "version": "7.11.1", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.11.1.tgz", "integrity": "sha512-ayO8XV0xuJV3cEY4wySyD/7MA1HL75UpvJ5JAme00kNWA5pddlGtN4BRG97xgGe2NHgwxN8AkdjTQUEDypM8Uw==", + "dev": true, "requires": { "@types/node": "*", "@types/pg-types": "*" @@ -191,6 +192,7 @@ "version": "1.11.4", "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.4.tgz", "integrity": "sha512-WdIiQmE347LGc1Vq3Ki8sk3iyCuLgnccqVzgxek6gEHp2H0p3MQ3jniIHt+bRODXKju4kNQ+mp53lmP5+/9moQ==", + "dev": true, "requires": { "moment": ">=2.14.0" } diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 578550f23..592f1138f 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -19,6 +19,7 @@ const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); const { NeDBDataStore } = require("../datastore/NedbDataStore"); +const { PgDataStore } = require("../datastore/postgres/PgDataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index ab939db6f..3ce1fbab5 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -335,7 +335,7 @@ export class NeDBDataStore implements DataStore { const entries: Entry[] = await this.roomStore.getEntriesByMatrixId(roomId); for (const entry of entries) { if (!entry.remote) { - return; + continue; } const modes = entry.remote.get("modes") as string[] || []; const hasMode = modes.includes(mode); diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 63916a9ba..fbbfd7154 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -18,8 +18,8 @@ import { Pool } from "pg"; // eslint-disable-next-line @typescript-eslint/no-duplicate-imports -import { MatrixUser, MatrixRoom, RemoteRoom } from "matrix-appservice-bridge"; -import { DataStore, RoomOrigin, ChannelMappings, RoomEntry, UserFeatures } from "../DataStore"; +import { MatrixUser, MatrixRoom, RemoteRoom, Entry } from "matrix-appservice-bridge"; +import { DataStore, RoomOrigin, ChannelMappings, UserFeatures } from "../DataStore"; import { IrcRoom } from "../../models/IrcRoom"; import { IrcClientConfig } from "../../models/IrcClientConfig"; import { IrcServer, IrcServerConfig } from "../../irc/IrcServer"; @@ -97,18 +97,20 @@ export class PgDataStore implements DataStore { await this.pgPool.query(statement, Object.values(parameters)); } - private static pgToRoomEntry(pgEntry: any): RoomEntry { + private static pgToRoomEntry(pgEntry: any): Entry { return { id: "", matrix: new MatrixRoom(pgEntry.room_id, JSON.parse(pgEntry.matrix_json)), remote: new RemoteRoom("", JSON.parse(pgEntry.irc_json)), + matrix_id: pgEntry.room_id, + remote_id: "foobar", data: { origin: pgEntry.origin, }, }; } - public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise { + public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise { let statement = "SELECT * FROM rooms WHERE room_id = $1, irc_domain = $2, irc_channel = $3"; if (origin) { statement += ", origin = $4"; @@ -145,13 +147,13 @@ export class PgDataStore implements DataStore { return mappings; } - public getEntriesByMatrixId(roomId: string): Bluebird { + public getEntriesByMatrixId(roomId: string): Bluebird { return Bluebird.cast(this.pgPool.query("SELECT * FROM rooms WHERE room_id = $1", [ roomId ])).then((result) => result.rows).map((e) => PgDataStore.pgToRoomEntry(e)); } - public getProvisionedMappings(roomId: string): Bluebird { + public getProvisionedMappings(roomId: string): Bluebird { return Bluebird.cast(this.pgPool.query("SELECT * FROM rooms WHERE room_id = $1 AND origin = 'provision'", [ roomId ])).then((result) => result.rows).map((e) => PgDataStore.pgToRoomEntry(e)); @@ -204,7 +206,7 @@ export class PgDataStore implements DataStore { return entries.rows.map((e) => new MatrixRoom(e.room_id, JSON.parse(e.matrix_json))); } - public async getMappingsForChannelByOrigin(server: IrcServer, channel: string, origin: "config" | "provision" | "alias" | "join" | RoomOrigin[], allowUnset: boolean): Promise { + public async getMappingsForChannelByOrigin(server: IrcServer, channel: string, origin: RoomOrigin | RoomOrigin[], allowUnset: boolean): Promise { const entries = await this.pgPool.query("SELECT * FROM rooms WHERE irc_domain = $1 AND irc_channel = $2 AND origin = $3", [ server.domain, @@ -214,8 +216,8 @@ export class PgDataStore implements DataStore { return entries.rows.map((e) => PgDataStore.pgToRoomEntry(e)); } - public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string; }> { - const mapping: {[id: string]: string} = {}; + public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string[]; }> { + const mapping: {[id: string]: string[]} = {}; const entries = await this.pgPool.query( "SELECT room_id, remote_json->>'modes' AS MODES FROM rooms " + "WHERE irc_domain = $1 AND irc_channel = $2", @@ -233,8 +235,11 @@ export class PgDataStore implements DataStore { log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", mode, roomId, enabled ); - const entries: RoomEntry[] = await this.getEntriesByMatrixId(roomId); + const entries: Entry[] = await this.getEntriesByMatrixId(roomId); for (const entry of entries) { + if (!entry.remote) { + continue; + } const modes = entry.remote.get("modes") as string[] || []; const hasMode = modes.includes(mode); @@ -306,7 +311,7 @@ export class PgDataStore implements DataStore { await this.pgPool.query("UPDATE ipv6_counter SET count = $1", [ counter ]); } - public async upsertRoomStoreEntry(entry: RoomEntry): Promise { + public async upsertRoomStoreEntry(entry: Entry): Promise { throw new Error("Method not implemented."); } From e9af1fa8037b728b0450c8c20a869364220ae039 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 17:20:10 +0100 Subject: [PATCH 021/350] Add migration script --- scripts/migrate-db-to-pgres.sh | 5 + src/scripts/migrate-db-to-pgres.ts | 212 +++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100755 scripts/migrate-db-to-pgres.sh create mode 100644 src/scripts/migrate-db-to-pgres.ts diff --git a/scripts/migrate-db-to-pgres.sh b/scripts/migrate-db-to-pgres.sh new file mode 100755 index 000000000..08f3d0356 --- /dev/null +++ b/scripts/migrate-db-to-pgres.sh @@ -0,0 +1,5 @@ +#!/bin/bash +SCRIPTPATH=`dirname $0` +SCRIPT=`realpath $SCRIPTPATH/../lib/scripts/migrate-db-to-pgres.js` + +node $SCRIPT "$@" \ No newline at end of file diff --git a/src/scripts/migrate-db-to-pgres.ts b/src/scripts/migrate-db-to-pgres.ts new file mode 100644 index 000000000..bb4200525 --- /dev/null +++ b/src/scripts/migrate-db-to-pgres.ts @@ -0,0 +1,212 @@ +import NeDB from "nedb"; +import nopt from "nopt"; +import { Logger, transports } from "winston"; +import path from "path"; +import { promises as fs } from "fs"; +import { formatterFn, timestampFn } from "../logging"; +import { promisify } from "util"; +import { PgDataStore } from "../datastore/postgres/PgDataStore"; +import { IrcRoom } from "../models/IrcRoom"; +import { MatrixRoom, MatrixUser } from "matrix-appservice-bridge"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +// Migrate rooms +// Migrate users +// Migrate configs + +const log = new Logger({ + level: "info", + transports: [ + new transports.Console({ + json: false, + name: "console", + timestamp: timestampFn, + formatter: formatterFn, + }) + ], +}); + +// NeDB is schemaless +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type promisfiedFind = (params: any) => Promise; + +async function migrate(roomsFind: promisfiedFind, usersFind: promisfiedFind, pgStore: PgDataStore) { + const migrateChannels = async () => { + const channelEntries = await roomsFind({ "remote.type": "channel" }); + log.info(`Migrating ${channelEntries.length} channels`); + for (const entry of channelEntries) { + await pgStore.upsertRoom( + entry.data.origin, + "channel", + entry.remote.domain, + entry.remote.channel, + entry.matrix_id, + JSON.stringify(entry.remote), + JSON.stringify(entry.matrix), + ); + } + log.info("Migrated channels"); + } + const migrateCounter = async () => { + log.info(`Migrating ipv6 counter`); + const counterEntry = await usersFind({ "type": "remote", "id": "config" }); + if (counterEntry.length && counterEntry[0].data && counterEntry[0].data.ipv6_counter) { + await pgStore.setIpv6Counter(counterEntry[0].data.ipv6_counter); + } + else { + log.info("No ipv6 counter found"); + } + log.info("Migrated ipv6 counter"); + } + const migrateAdminRooms = async () => { + const entries = await roomsFind({ "matrix.extras.admin_id": { $exists: true } }); + log.info(`Migrating ${entries.length} admin rooms`); + for (const entry of entries) { + await pgStore.storeAdminRoom( + new MatrixRoom(entry.matrix_id), + entry.matrix.extras.admin_id, + ); + } + log.info("Migrated admin rooms"); + } + + const migrateUserFeatures = async () => { + const entries = await usersFind({ "type": "matrix", "data.features": { $exists: true } }); + log.info(`Migrating ${entries.length} user features`); + for (const entry of entries) { + await pgStore.storeUserFeatures( + entry.id, + entry.data.features, + ); + } + log.info("Migrated user features"); + } + + const migrateUserConfiguration = async () => { + const entries = await usersFind({ "type": "matrix", "data.client_config": { $exists: true } }); + log.info(`Migrating ${entries.length} user configs`); + for (const entry of entries) { + const configs = entry.data.client_config; + for (const network of Object.keys(configs)) { + const config = configs[network]; + const password = config.password; + await pgStore.storeIrcClientConfig(new IrcClientConfig(entry.id, network, config)); + await pgStore.storePass(entry.id, network, password, false); + } + await pgStore.storeUserFeatures( + entry.id, + entry.data.features, + ); + } + log.info("Migrated user configs"); + } + + const migrateUsers = async () => { + const entries = await usersFind({ "type": "matrix" }); + log.info(`Migrating ${entries.length} users`); + for (const entry of entries) { + // We store these seperately. + delete entry.data.client_config; + delete entry.data.features; + await pgStore.storeMatrixUser( + new MatrixUser(entry.id, entry.data) + ); + } + log.info("Migrated users"); + } + + const migratePMs = async () => { + const entries = await roomsFind({ "remote.type": "pm" }); + log.info(`Migrating ${entries.length} PM rooms`); + for (const entry of entries) { + // We store these seperately. + await pgStore.setPmRoom( + // IrcRoom will only ever use the domain property + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new IrcRoom({ domain: entry.remote.domain } as any, entry.remote.channel), + new MatrixRoom(entry.matrix_id), + entry.data.real_user_id, + entry.data.virtual_user_id, + ); + } + log.info("Migrated users"); + } + + await migrateChannels(); + await migrateCounter(); + await migrateAdminRooms(); + await migrateUserFeatures(); + await migrateUserConfiguration(); + await migrateUsers(); + await migratePMs(); +} + +async function main() { + const opts = nopt({ + "dbdir": path, + "connectionString": String, + "verbose": Boolean, + "privateKey": path + }, + { + "f": "--dbdir", + "c": "--connectionString", + "p": "--privateKey", + "v": "--verbose" + }, process.argv, 2); + + if (opts.dbdir === undefined || opts.connectionString === undefined) { + log.error("Missing --dbdir or --connectionString or --domain"); + process.exit(1); + } + + if (opts.privateKey === undefined) { + log.warn("Missing privateKey, passwords will not be migrated"); + } + + if (opts.verbose) { + log.level = "verbose"; + } + + try { + await Promise.all(["rooms.db", "users.db"].map(async f => { + const p = path.join(opts.dbdir, f); + const stats = await fs.stat(p); + if (stats.isDirectory() && stats.size > 0) { + throw Error(`${p} must be a file`); + } + })); + } + catch (ex) { + log.error("Missing a file: %s", ex); + process.exit(1); + } + + // Domain isn't used for any of our operations + const pgStore = new PgDataStore("", opts.connectionString, opts.privateKey); + + try { + await pgStore.ensureSchema(); + } + catch (ex) { + log.warn("Could not ensure schema version: %s", ex); + process.exit(1); + } + + const rooms = new NeDB({ filename: path.join(opts.dbdir, "rooms.db"), autoload: true }); + const users = new NeDB({ filename: path.join(opts.dbdir, "users.db"), autoload: true }); + + const roomsFind = promisify(rooms.find).bind(rooms) as promisfiedFind; + const usersFind = promisify(users.find).bind(users) as promisfiedFind; + + const time = Date.now(); + log.info("Starting migration"); + await migrate(roomsFind, usersFind, pgStore); + log.info("Finished migration at %sms", Date.now() - time); +} + +main().then(() => { + +}).catch((ex) => { + log.error("Failed to run migration script: %s", ex); +}) From 6e37728bc01a4c041fb35c00f401b0961f850366 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 19 Sep 2019 17:20:56 +0100 Subject: [PATCH 022/350] Cleanup --- .ts.eslintrc | 1 - package-lock.json | 6 ++++++ package.json | 1 + src/datastore/postgres/PgDataStore.ts | 1 - src/logging.js | 28 +++++++++++++++------------ src/models/IrcClientConfig.ts | 2 -- src/models/IrcRoom.ts | 6 +----- 7 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.ts.eslintrc b/.ts.eslintrc index d0c9fbcae..d10472e47 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -3,7 +3,6 @@ "plugins": ["@typescript-eslint"], "extends": ["plugin:@typescript-eslint/recommended", ".eslintrc"], "rules": { - "@typescript-eslint/ban-ts-ignore": 0, "@typescript-eslint/explicit-function-return-type": 0, "@typescript-eslint/camelcase": ["error", { "properties": "never" }] } diff --git a/package-lock.json b/package-lock.json index b6bdf8fc7..76f58b049 100644 --- a/package-lock.json +++ b/package-lock.json @@ -178,6 +178,12 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" }, + "@types/nopt": { + "version": "3.0.29", + "resolved": "https://registry.npmjs.org/@types/nopt/-/nopt-3.0.29.tgz", + "integrity": "sha1-8Z3z20yX7hRZonQAKDIKcdcJZM4=", + "dev": true + }, "@types/pg": { "version": "7.11.1", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-7.11.1.tgz", diff --git a/package.json b/package.json index 95c33d121..57395934b 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@types/bluebird": "^3.5.27", "@types/nedb": "^1.8.9", "@types/pg": "^7.11.1", + "@types/nopt": "^3.0.29", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index fbbfd7154..5424b5fd6 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -26,7 +26,6 @@ import { IrcServer, IrcServerConfig } from "../../irc/IrcServer"; import * as logging from "../../logging"; import Bluebird from "bluebird"; -import { stat } from "fs"; import { StringCrypto } from "../StringCrypto"; const log = logging.get("PgDatastore"); diff --git a/src/logging.js b/src/logging.js index d12d7c609..2615a1811 100644 --- a/src/logging.js +++ b/src/logging.js @@ -20,18 +20,19 @@ var loggers = { }; var loggerTransports; // from config +const timestampFn = function() { + return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); +}; +const formatterFn = function(opts) { + return opts.timestamp() + ' ' + + opts.level.toUpperCase() + ':' + + (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + + (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + + (opts.meta && opts.meta.dir ? opts.meta.dir : "") + + (undefined !== opts.message ? opts.message : ''); +}; + var makeTransports = function() { - var timestampFn = function() { - return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); - }; - var formatterFn = function(opts) { - return opts.timestamp() + ' ' + - opts.level.toUpperCase() + ':' + - (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + - (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + - (opts.meta && opts.meta.dir ? opts.meta.dir : "") + - (undefined !== opts.message ? opts.message : ''); - }; var transports = []; if (loggerConfig.toConsole) { @@ -211,5 +212,8 @@ module.exports = { } }); }); - } + }, + + timestampFn, + formatterFn, }; diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index bc2883942..3f712c12c 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Ignore definition errors for now. -//@ts-ignore import { MatrixUser } from "matrix-appservice-bridge"; export interface IrcClientConfigSeralized { diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index e51ac7e18..38d89891b 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -//@ts-ignore import { RemoteRoom } from "matrix-appservice-bridge"; import { toIrcLowerCase } from "../irc/formatting"; import { IrcServer } from "../irc/IrcServer"; @@ -26,7 +25,6 @@ export class IrcRoom extends RemoteRoom { * @param {IrcServer} server : The IRC server which contains this room. * @param {String} channel : The channel this room represents. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(public readonly server: IrcServer, public readonly channel: string) { // Because `super` must be called first, we convert the case several times. super(IrcRoom.createId(server, toIrcLowerCase(channel)), { @@ -56,9 +54,7 @@ export class IrcRoom extends RemoteRoom { return super.get("type") as string; } - // No types for IrcServer yet - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public static fromRemoteRoom(server: any, remoteRoom: RemoteRoom) { + public static fromRemoteRoom(server: IrcServer, remoteRoom: RemoteRoom) { return new IrcRoom(server, remoteRoom.get("channel") as string); } From 9a90742cd160ea9708a7b8b1d33489e73f492996 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Sep 2019 11:12:40 +0100 Subject: [PATCH 023/350] Replace calls to upsertRoomStoreEntry with upsertMatrixRoom --- src/bridge/IrcHandler.js | 2 +- src/datastore/DataStore.ts | 2 +- src/datastore/NedbDataStore.ts | 4 ++-- src/datastore/postgres/PgDataStore.ts | 10 ++++++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index a3e7b3fc0..0dfbea8a0 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -420,7 +420,7 @@ IrcHandler.prototype._serviceTopicQueue = Promise.coroutine(function*(item) { }).then( () => { entry.matrix.topic = item.topic; - return this.ircBridge.getStore().upsertRoomStoreEntry(entry); + return this.ircBridge.getStore().upsertMatrixRoom(entry.matrix); }, (err) => { item.req.log.error(`Error storing room ${entry.matrix.getId()} (${err.message})`); diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index b051957f1..0253c3fbb 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -120,7 +120,7 @@ export interface DataStore { storeAdminRoom(room: MatrixRoom, userId: string): Promise; - upsertRoomStoreEntry(entry: Entry): Promise; + upsertMatrixRoom(room: MatrixRoom): Promise; getAdminRoomByUserId(userId: string): Promise; diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 3ce1fbab5..3df73cbfa 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -475,8 +475,8 @@ export class NeDBDataStore implements DataStore { }); } - public async upsertRoomStoreEntry(entry: Entry): Promise { - await this.roomStore.upsertEntry(entry); + public async upsertMatrixRoom(room: MatrixRoom): Promise { + await this.roomStore.setMatrixRoom(room); } public async getAdminRoomByUserId(userId: string): Promise { diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index fbbfd7154..6f71f384f 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -311,8 +311,14 @@ export class PgDataStore implements DataStore { await this.pgPool.query("UPDATE ipv6_counter SET count = $1", [ counter ]); } - public async upsertRoomStoreEntry(entry: Entry): Promise { - throw new Error("Method not implemented."); + public async upsertMatrixRoom(room: MatrixRoom): Promise { + // XXX: This is an upsert operation, but we don't have enough details to go on + // so this will just update a rooms data entry. We only use this call to update + // topics on an existing room. + await this.pgPool.query("UPDATE rooms SET matrix_json = $1 WHERE room_id = $2", [ + JSON.stringify(room.serialize()), + room.getId(), + ]); } public async getAdminRoomById(roomId: string): Promise { From 8785d3a6dd292a262878c35a88577a7b9d159f97 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Sep 2019 14:38:33 +0100 Subject: [PATCH 024/350] New config options --- config.sample.yaml | 14 +++++++++----- src/bridge/IrcBridge.js | 23 +++++++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 7dbbef3dc..b8c8bba53 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -391,11 +391,6 @@ ircService: - "1d" - "1w" - # The nedb database URI to connect to. This is the name of the directory to - # dump .db files to. This is relative to the project directory. - # Required. - databaseUri: "nedb://data" - # Configuration options for the debug HTTP API. To access this API, you must # append ?access_token=$APPSERVICE_TOKEN (from the registration file) to the requests. # @@ -469,3 +464,12 @@ advanced: # accidentally overloading the homeserver. Defaults to 1000, which should be # enough for the vast majority of use cases. maxHttpSockets: 1000 + +# Use an external database to store bridge state. +database: + # database engine (must be 'postgres' or 'nedb'). Default: nedb + engine: "postgres" + # Either a PostgreSQL connection string, or a path to the NeDB storage directory. + # For postgres, it must start with postgres:// + # For NeDB, it must start with nedb://. The path is relative to the project directory. + connectionString: "postgres://username:password@host:port/databasename" \ No newline at end of file diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 592f1138f..1a983af30 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -67,7 +67,13 @@ function IrcBridge(config, registration) { this.matrixHandler = new MatrixHandler(this, this.config.matrixHandler); this.ircHandler = new IrcHandler(this, this.config.ircHandler); this._clientPool = new ClientPool(this); - var dirPath = this.config.ircService.databaseUri.substring("nedb://".length); + if (!this.config.database && this.config.ircService.databaseUri) { + log.warn("ircService.databaseUri is a deprecated config option. Please use the database configuration block"); + this.config.database = { + engine: "nedb", + connectionString: this.config.ircService.databaseUri, + } + } let roomLinkValidation = undefined; let provisioning = config.ircService.provisioning; if (provisioning && provisioning.enabled && @@ -78,6 +84,16 @@ function IrcBridge(config, registration) { }; } + let bridgeStoreConfig = {}; + + if (this.confg.database.engine === "nedb") { + const dirPath = this.config.database.connectionString.substring("nedb://".length); + bridgeStoreConfig = { + roomStore: `${dirPath}/rooms.db`, + userStore: `${dirPath}/users.db`, + }; + } + this._bridge = new Bridge({ registration: this.registration, homeserverUrl: this.config.homeserver.url, @@ -97,8 +113,7 @@ function IrcBridge(config, registration) { getUser: this.getThirdPartyUser.bind(this), }, }, - roomStore: dirPath + "/rooms.db", - userStore: dirPath + "/users.db", + ...bridgeStoreConfig, disableContext: true, suppressEcho: false, // we use our own dupe suppress for now logRequestOutcome: false, // we use our own which has better logging @@ -319,7 +334,7 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; - const dbConfig = this.config.ircService.database; + const dbConfig = this.config.database; if (dbConfig && dbConfig.engine === "postgres") { this._dataStore = new PgDataStore(this.config.homeserver.domain, dbConfig.connectionString, pkeyPath); yield this._dataStore.ensureSchema(); From 88f5f518b9082bd482299590b9cc6592b98478c3 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Sep 2019 14:38:42 +0100 Subject: [PATCH 025/350] Fix broken schema --- src/datastore/postgres/schema/v1.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/datastore/postgres/schema/v1.ts b/src/datastore/postgres/schema/v1.ts index d58b02419..d0431b269 100644 --- a/src/datastore/postgres/schema/v1.ts +++ b/src/datastore/postgres/schema/v1.ts @@ -40,7 +40,7 @@ export async function runSchema(connection: PoolClient) { CREATE TABLE matrix_users ( user_id TEXT UNIQUE, - data TEXT + data JSON ); CREATE TABLE client_config ( @@ -60,16 +60,5 @@ export async function runSchema(connection: PoolClient) { count INTEGER ); - CREATE TABLE ( - origin TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - irc_domain TEXT NOT NULL, - irc_channel TEXT NOT NULL, - irc_json JSON NOT NULL, - matrix_json JSON NOT NULL, - CONSTRAINT cons_rooms_unique UNIQUE(room_id, irc_domain, irc_channel) - ); - INSERT INTO ipv6_counter VALUES (0);`); } From af0442686f83da2c8cb54a889bad1258ad9965ae Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Sep 2019 14:39:17 +0100 Subject: [PATCH 026/350] Tweaks to PgDatastore --- src/datastore/postgres/PgDataStore.ts | 64 +++++++++++++++++++-------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 6f71f384f..8b713b4d9 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -72,13 +72,17 @@ export class PgDataStore implements DataStore { } log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin); + const ircRoomSerial = ircRoom.serialize() as any; + delete ircRoomSerial.domain; + delete ircRoomSerial.channel; + delete ircRoomSerial.type; this.upsertRoom( origin, ircRoom.getType(), ircRoom.getDomain(), ircRoom.getChannel(), matrixRoom.getId(), - JSON.stringify(ircRoom.serialize()), + JSON.stringify(ircRoomSerial), JSON.stringify(matrixRoom.serialize()), ); } @@ -100,8 +104,14 @@ export class PgDataStore implements DataStore { private static pgToRoomEntry(pgEntry: any): Entry { return { id: "", - matrix: new MatrixRoom(pgEntry.room_id, JSON.parse(pgEntry.matrix_json)), - remote: new RemoteRoom("", JSON.parse(pgEntry.irc_json)), + matrix: new MatrixRoom(pgEntry.room_id, pgEntry.matrix_json), + remote: new RemoteRoom("", + { + ...pgEntry.irc_json, + channel: pgEntry.irc_channel, + domain: pgEntry.irc_domain, + type: pgEntry.type, + }), matrix_id: pgEntry.room_id, remote_id: "foobar", data: { @@ -167,14 +177,18 @@ export class PgDataStore implements DataStore { } public async getIrcChannelsForRoomId(roomId: string): Promise { - const entries = await this.pgPool.query("SELECT irc_domain, irc_channel FROM rooms WHERE room_id = $1", [ roomId ]); + let entries = await this.pgPool.query("SELECT irc_domain, irc_channel FROM rooms WHERE room_id = $1", [ roomId ]); + if (entries.rowCount === 0) { + // Could be a PM room, if it's not a channel. + entries = await this.pgPool.query("SELECT irc_domain, irc_nick FROM pm_rooms WHERE room_id = $1", [ roomId ]); + } return entries.rows.map((e) => { const server = this.serverMappings[e.irc_domain]; if (!server) { // ! is used here because typescript doesn't understand the .filter return undefined!; } - return new IrcRoom(server, e.irc_channel); + return new IrcRoom(server, e.irc_channel || e.irc_nick); }).filter((i) => i !== undefined); } @@ -203,30 +217,36 @@ export class PgDataStore implements DataStore { server.domain, channel, ]); - return entries.rows.map((e) => new MatrixRoom(e.room_id, JSON.parse(e.matrix_json))); + return entries.rows.map((e) => new MatrixRoom(e.room_id, e.matrix_json)); } public async getMappingsForChannelByOrigin(server: IrcServer, channel: string, origin: RoomOrigin | RoomOrigin[], allowUnset: boolean): Promise { - const entries = await this.pgPool.query("SELECT * FROM rooms WHERE irc_domain = $1 AND irc_channel = $2 AND origin = $3", + if (!Array.isArray(origin)) { + origin = [origin]; + } + const inStatement = origin.map((_, i) => `\$${i + 3}`).join(", "); + const statement = `SELECT * FROM rooms WHERE irc_domain = $1 AND irc_channel = $2 AND origin IN (${inStatement})`; + console.log(statement); + const entries = await this.pgPool.query(statement, [ server.domain, channel, - origin, - ]); + ].concat(origin)); return entries.rows.map((e) => PgDataStore.pgToRoomEntry(e)); } public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string[]; }> { + log.debug(`Getting modes for ${server.domain} ${channel}`); const mapping: {[id: string]: string[]} = {}; const entries = await this.pgPool.query( - "SELECT room_id, remote_json->>'modes' AS MODES FROM rooms " + + "SELECT room_id, irc_json->>'modes' AS modes FROM rooms " + "WHERE irc_domain = $1 AND irc_channel = $2", [ server.domain, channel, ]); entries.rows.forEach((e) => { - mapping[e.room_id] = e.modes; + mapping[e.room_id] = e.modes || []; }); return mapping; } @@ -254,11 +274,15 @@ export class PgDataStore implements DataStore { } entry.remote.set("modes", modes); + const ircRoomSerial = entry.remote.serialize() as any; + delete ircRoomSerial.domain; + delete ircRoomSerial.channel; + delete ircRoomSerial.type; await this.pgPool.query("UPDATE rooms WHERE room_id = $1, irc_channel = $2, irc_domain = $3 SET irc_json = $4", [ roomId, entry.remote.get("channel"), entry.remote.get("domain"), - JSON.stringify(entry.remote.serialize()), + JSON.stringify(ircRoomSerial), ]); } } @@ -285,11 +309,13 @@ export class PgDataStore implements DataStore { } public async getTrackedChannelsForServer(domain: string): Promise { - if (this.serverMappings[domain]) { + if (!this.serverMappings[domain]) { + // Return empty if we don't know the server. return []; } - const chanSet = await this.pgPool.query("SELECT channel FROM rooms WHERE irc_domain = $1", [ domain ]); - return [...new Set((chanSet.rows).map((e) => e.channel))]; + log.info(`Fetching all channels for ${domain}`); + const chanSet = await this.pgPool.query("SELECT DISTINCT irc_channel FROM rooms WHERE irc_domain = $1", [ domain ]); + return chanSet.rows.map((e) => e.irc_channel as string); } public async getRoomIdsFromConfig(): Promise { @@ -363,7 +389,7 @@ export class PgDataStore implements DataStore { return null; } const row = res.rows[0]; - let config = JSON.parse(row.config); + const config = row.config; if (row.password && this.cryptoStore) { config.password = this.cryptoStore.decrypt(row.password); } @@ -398,7 +424,7 @@ export class PgDataStore implements DataStore { return null; } const row = res.rows[0]; - return new MatrixUser(row.user_id, JSON.parse(row.data)); + return new MatrixUser(row.user_id, row.data); } public async getUserFeatures(userId: string): Promise { @@ -443,14 +469,14 @@ export class PgDataStore implements DataStore { public async getMatrixUserByUsername(domain: string, username: string): Promise { // This will need a join - const res = await this.pgPool.query("SELECT client_config.user_id, matrix_users.data FROM client_config, matrix_users" + + const res = await this.pgPool.query("SELECT client_config.user_id, matrix_users.data FROM client_config, matrix_users " + "WHERE config->>'username' = $1 AND domain = $2 AND client_config.user_id = matrix_users.user_id", [username, domain] ); if (res.rowCount === 0) { return; } - return new MatrixUser(res.rows[0].user_id, JSON.parse(res.rows[0].data)); + return new MatrixUser(res.rows[0].user_id, res.rows[0].data); } public async ensureSchema() { From 63b88e3d920ceff07863038a1040d013496c8572 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Sep 2019 14:55:43 +0100 Subject: [PATCH 027/350] Move + update config schema --- app.js | 2 +- config.schema.yml | 330 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 config.schema.yml diff --git a/app.js b/app.js index da9412cd1..cc8d3c4f5 100644 --- a/app.js +++ b/app.js @@ -12,7 +12,7 @@ new Cli({ enableLocalpart: true, bridgeConfig: { affectsRegistration: true, - schema: path.join(__dirname, "lib/config/schema.yml"), + schema: path.join(__dirname, "config.schema.yml"), defaults: { homeserver: { dropMatrixMessagesAfterSecs: 0, diff --git a/config.schema.yml b/config.schema.yml new file mode 100644 index 000000000..e2773111f --- /dev/null +++ b/config.schema.yml @@ -0,0 +1,330 @@ +"$schema": "http://json-schema.org/draft-04/schema#" +type: "object" +properties: + advanced: + type: "object" + properties: + maxHttpSockets: + type: "integer" + database: + type: "object" + required: ["engine", "connectionString"] + properties: + engine: + type: "string" + enum: ["postgres", "nedb"] + connectionString: + type: "string" + homeserver: + type: "object" + properties: + url: + type: "string" + media_url: + type: "string" + domain: + type: "string" + dropMatrixMessagesAfterSecs: + type: "integer" + enablePresence: + type: "boolean" + required: ["url", "domain"] + ircService: + type: "object" + properties: + metrics: + type: "object" + properties: + enabled: + type: "boolean" + remoteUserAgeBuckets: + type: "array" + items: + type: "string" + pattern: "^[0-9]+(h|d|w)$" + statsd: + type: "object" + properties: + hostname: + type: "string" + port: + type: "integer" + jobName: + type: "string" + required: ["hostname", "port"] + ident: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + address: + type: "string" + required: ["enabled"] + debugApi: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + required: ["enabled", "port"] + logging: + type: "object" + properties: + level: + type: "string" + enum: ["error","warn","info","debug"] + logfile: + type: "string" + errfile: + type: "string" + toConsole: + type: "boolean" + maxFileSizeBytes: + type: "integer" + maxFiles: + type: "integer" + provisioning: + type: "object" + properties: + enabled: + type: "boolean" + requestTimeoutSeconds: + type: "number" + ruleFile: + type: "string" + enableReload: + type: "boolean" + passwordEncryptionKeyPath: + type: "string" + matrixHandler: + type: "object" + properties: + eventCacheSize: + type: "integer" + ircHandler: + type: "object" + properties: + leaveConcurrency: + type: "integer" + mapIrcMentionsToMatrix: + type: "string" + enum: ["on", "off", "force-off"] + servers: + type: "object" + # all properties must follow the following + additionalProperties: + type: "object" + properties: + port: + type: "integer" + additionalAddresses: + type: "array" + items: + type: "string" + ssl: + type: "boolean" + sslselfsign: + type: "boolean" + sasl: + type: "boolean" + allowExpiredCerts: + type: "boolean" + password: + type: "string" + sendConnectionMessages: + type: "boolean" + name: + type: "string" + description: + type: "string" + networkId: + type: "string" + pattern: "^[a-zA-Z0-9]+$" + icon: + type: "string" + quitDebounce: + type: "object" + properties: + enabled: + type: "boolean" + quitsPerSecond: + type: "number" + delayMinMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + delayMaxMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + modePowerMap: + type: "object" + patternProperties: + # Single character modes mapped to positive power levels + "^[a-zA-Z]$": + type: number + minimum: 0 + botConfig: + type: "object" + properties: + enabled: + type: "boolean" + nick: + type: "string" + password: + type: "string" + joinChannelsIfNoUsers: + type: "boolean" + privateMessages: + type: "object" + properties: + enabled: + type: "boolean" + exclude: + type: "array" + items: + type: "string" + federate: + type: "boolean" + membershipLists: + type: "object" + properties: + enabled: + type: "boolean" + floodDelayMs: + type: "integer" + global: + type: "object" + properties: + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + rooms: + type: "array" + items: + type: "object" + properties: + room: + type: "string" + pattern: "^!+.*$" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + channels: + type: "array" + items: + type: "object" + properties: + channel: + type: "string" + pattern: "^#+.*$" + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + dynamicChannels: + type: "object" + properties: + enabled: + type: "boolean" + published: + type: "boolean" + createAlias: + type: "boolean" + groupId: + type: "string" + joinRule: + type: "string" + enum: ["invite", "public"] + federate: + type: "boolean" + roomVersion: + type: "string" + aliasTemplate: + type: "string" + pattern: "^#.*\\$CHANNEL" + whitelist: + type: "array" + items: + type: "string" + pattern: "^@.*" + exclude: + type: "array" + items: + type: "string" + mappings: + type: "object" + patternProperties: + # must start with a # + "^#+.*$": + type: "array" + items: + type: "string" + minItems: 1 + uniqueItems: true + additionalProperties: false + matrixClients: + type: "object" + properties: + userTemplate: + type: "string" + pattern: "^@.*\\$NICK" + displayName: + type: "string" + pattern: "\\$NICK" + joinAttempts: + type: "integer" + minimum: -1 + ircClients: + type: "object" + properties: + nickTemplate: + type: "string" + pattern: "\\$USERID|\\$LOCALPART|\\$DISPLAY" + maxClients: + type: "integer" + idleTimeout: + type: "integer" + minimum: 0 + reconnectIntervalMs: + type: "integer" + minimum: 0 + allowNickChanges: + type: "boolean" + ipv6: + type: "object" + properties: + prefix: + type: "string" + pattern: "[ABCDEFabcdef0123456789:]+" + only: + type: "boolean" + lineLimit: + type: "integer" + userModes: + type: "string" + required: ["servers"] From 2d3abd907f2b4f7a9dc09c4f87d694c24935e7fb Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Sun, 22 Sep 2019 19:29:06 +0100 Subject: [PATCH 028/350] Convert BridgeRequest to Typescript --- src/DebugApi.js | 2 +- src/bridge/IrcBridge.js | 2 +- src/bridge/IrcHandler.js | 2 +- src/bridge/MatrixHandler.js | 2 +- src/irc/IrcEventBroker.js | 2 +- src/models/BridgeRequest.js | 28 ---------------------- src/models/BridgeRequest.ts | 42 +++++++++++++++++++++++++++++++++ src/provisioning/Provisioner.js | 2 +- 8 files changed, 48 insertions(+), 34 deletions(-) delete mode 100644 src/models/BridgeRequest.js create mode 100644 src/models/BridgeRequest.ts diff --git a/src/DebugApi.js b/src/DebugApi.js index 0942c630c..311283eae 100644 --- a/src/DebugApi.js +++ b/src/DebugApi.js @@ -2,7 +2,7 @@ "use strict"; const querystring = require("querystring"); const Promise = require("bluebird"); -const BridgeRequest = require("./models/BridgeRequest"); +const { BridgeRequest } = require("./models/BridgeRequest"); const log = require("./logging").get("DebugApi"); const http = require("http"); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index b960fd21d..d0b71c755 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -16,7 +16,7 @@ var BridgedClient = require("../irc/BridgedClient"); var IrcUser = require("../models/IrcUser"); const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); -var BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); var stats = require("../config/stats"); const { DataStore } = require("../DataStore"); var log = require("../logging").get("IrcBridge"); diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index a3e7b3fc0..ceb046cc6 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -2,7 +2,7 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index a9ad259b5..0459aa01b 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -9,7 +9,7 @@ const MatrixAction = require("../models/MatrixAction"); const IrcAction = require("../models/IrcAction"); const { IrcClientConfig } = require("../models/IrcClientConfig"); const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const toIrcLowerCase = require("../irc/formatting").toIrcLowerCase; const StateLookup = require('matrix-appservice-bridge').StateLookup; diff --git a/src/irc/IrcEventBroker.js b/src/irc/IrcEventBroker.js index 4fe08c8c4..983a87d94 100644 --- a/src/irc/IrcEventBroker.js +++ b/src/irc/IrcEventBroker.js @@ -69,7 +69,7 @@ "use strict"; const IrcAction = require("../models/IrcAction"); const IrcUser = require("../models/IrcUser"); -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const log = require("../logging").get("IrcEventBroker"); const CLEANUP_TIME_MS = 1000 * 60 * 10; // 10min diff --git a/src/models/BridgeRequest.js b/src/models/BridgeRequest.js deleted file mode 100644 index 0e801e34d..000000000 --- a/src/models/BridgeRequest.js +++ /dev/null @@ -1,28 +0,0 @@ -"use strict"; -const logging = require("../logging"); -var log = logging.get("req"); - -class BridgeRequest { - constructor(req) { - this.req = req; - var isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; - this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); - } - - getPromise() { - return this.req.getPromise(); - } - - resolve(thing) { - this.req.resolve(thing); - } - - reject(err) { - this.req.reject(err); - } -} -BridgeRequest.ERR_VIRTUAL_USER = "virtual-user"; -BridgeRequest.ERR_NOT_MAPPED = "not-mapped"; -BridgeRequest.ERR_DROPPED = "dropped"; - -module.exports = BridgeRequest; diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts new file mode 100644 index 000000000..4c57b281b --- /dev/null +++ b/src/models/BridgeRequest.ts @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const logging = require("../logging"); +const log = logging.get("req"); + +export class BridgeRequest { + private log: any; + constructor(private req: any) { + const isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; + this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); + } + + getPromise() { + return this.req.getPromise(); + } + + resolve(thing: any) { + this.req.resolve(thing); + } + + reject(err: any) { + this.req.reject(err); + } + + public static ERR_VIRTUAL_USER = "virtual-user"; + public static ERR_NOT_MAPPED = "virtual-user"; + public static ERR_DROPPED = "virtual-user"; +} \ No newline at end of file diff --git a/src/provisioning/Provisioner.js b/src/provisioning/Provisioner.js index 6a2f545c4..689d1d8cc 100644 --- a/src/provisioning/Provisioner.js +++ b/src/provisioning/Provisioner.js @@ -6,7 +6,7 @@ const IrcAction = require("../models/IrcAction"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const ConfigValidator = require("matrix-appservice-bridge").ConfigValidator; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const ProvisionRequest = require("./ProvisionRequest"); const log = require("../logging").get("Provisioner"); From 14bdf796126e65faf6dac7302e65a03d67ed148c Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Sun, 22 Sep 2019 19:32:10 +0100 Subject: [PATCH 029/350] Initial attempt at converting ClientPool to typescript --- src/irc/{ClientPool.js => ClientPool.ts} | 121 ++++++++++++----------- 1 file changed, 61 insertions(+), 60 deletions(-) rename src/irc/{ClientPool.js => ClientPool.ts} (87%) diff --git a/src/irc/ClientPool.js b/src/irc/ClientPool.ts similarity index 87% rename from src/irc/ClientPool.js rename to src/irc/ClientPool.ts index 4858dc816..1dcd79af9 100644 --- a/src/irc/ClientPool.js +++ b/src/irc/ClientPool.ts @@ -1,55 +1,56 @@ -/*eslint no-invalid-this: 0*/ /* - * Maintains a lookup of connected IRC clients. These connections are transient - * and may be closed for a variety of reasons. - */ -"use strict"; +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + const stats = require("../config/stats"); const log = require("../logging").get("ClientPool"); const Promise = require("bluebird"); const QueuePool = require("../util/QueuePool"); -const BridgeRequest = require("../models/BridgeRequest"); +import { BridgeRequest } from "../models/BridgeRequest"; -class ClientPool { - constructor(ircBridge) { - this._ircBridge = ircBridge; +/* + * Maintains a lookup of connected IRC clients. These connections are transient + * and may be closed for a variety of reasons. + */ +export class ClientPool { + private botClients: { [serverDomain: string]: any}; + private virtualClients: { [serverDomain: string]: { + nicks: { [nickname: string]: any}, + userIds: { [userId: string]: any}, + pending: { [nick: string]: any}, + }}; + private virtualClientCounts: { [serverDomain: string]: number }; + private reconnectQueues: { [serverDomain: string]: any }; + constructor(private ircBridge: any) { // The list of bot clients on servers (not specific users) - this._botClients = { - // server_domain: BridgedClient - }; + this.botClients = { }; // list of virtual users on servers - this._virtualClients = { - // server_domain: { - // nicks: { - // : BridgedClient - // }, - // userIds: { - // : BridgedClient - // } - // These users are in the process of being - // connected with an *assumed* nick. - // pending: { - // - // } - // } - } + this.virtualClients = { }; // map of numbers of connected clients on each server // Counting these is quite expensive because we have to // ignore entries where the value is undefined. Instead, // just keep track of how many we have. - this._virtualClientCounts = { - // server_domain: number - }; + this.virtualClientCounts = { }; - this._reconnectQueues = { - // server_domain: QueuePool - }; + this.reconnectQueues = { }; } - nickIsVirtual(server, nick) { - if (!this._virtualClients[server.domain]) { + public nickIsVirtual(server: any, nick: string) { + if (!this.virtualClients[server.domain]) { return false; } @@ -61,34 +62,34 @@ class ClientPool { const pending = Object.keys(this._virtualClients[server.domain].pending || {}); return pending.includes(nick); } -} -ClientPool.prototype.killAllClients = function() { - let domainList = Object.keys(this._virtualClients); - let clients = []; - domainList.forEach((domain) => { - clients = clients.concat( - Object.keys(this._virtualClients[domain].nicks).map( - (nick) => this._virtualClients[domain].nicks[nick] - ) - ); - - clients = clients.concat( - Object.keys(this._virtualClients[domain].userIds).map( - (userId) => this._virtualClients[domain].userIds[userId] + public killAllClients() { + const domainList = Object.keys(this.virtualClients); + let clients: any[] = []; + domainList.forEach((domain) => { + clients = clients.concat( + Object.keys(this.virtualClients[domain].nicks).map( + (nick) => this.virtualClients[domain].nicks[nick] + ) + ); + + clients = clients.concat( + Object.keys(this.virtualClients[domain].userIds).map( + (userId) => this.virtualClients[domain].userIds[userId] + ) + ); + + clients.push(this.botClients[domain]); + }); + + clients = clients.filter((c) => Boolean(c)); + + return Promise.all( + clients.map( + (client) => client.kill() ) ); - - clients.push(this._botClients[domain]); - }); - - clients = clients.filter((c) => Boolean(c)); - - return Promise.all( - clients.map( - (client) => client.kill() - ) - ); + } } ClientPool.prototype.getOrCreateReconnectQueue = function(server) { From 924ef6d8ca3311122e90bee0f5a01a16f2f5c176 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 09:28:39 +0100 Subject: [PATCH 030/350] WIP build out parts for testing postgres on CI --- .buildkite/pipeline.yml | 7 +++++ spec/util/test.js | 52 ++++++++++++++++++++++++++++++-------- src/bridge/IrcBridge.js | 2 +- src/datastore/DataStore.ts | 2 ++ 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1923243cc..ae965c753 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -25,6 +25,13 @@ steps: - docker#v3.0.1: image: "node:12" + - label: ":nodejs: 12 :postgres: 11 :mocha: Postgres Test" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.postgres.yml + - label: ":nyc: Coverage" command: - "npm install" diff --git a/spec/util/test.js b/spec/util/test.js index 1818915ea..df09278ad 100644 --- a/spec/util/test.js +++ b/spec/util/test.js @@ -6,6 +6,8 @@ const MockAppService = require("./app-service-mock"); const Promise = require("bluebird"); const clientMock = require("./client-sdk-mock"); const ircMock = require("./irc-client-mock"); +const { Client } = require('pg'); + clientMock["@global"] = true; ircMock["@global"] = true; const main = proxyquire("../../lib/main.js", { @@ -33,6 +35,16 @@ class TestEnv { this.ircBridge = null; this.ircMock = ircMock; this.clientMock = clientMock; + this.usingPg = process.env.IRCBRIDGE_TEST_ENABLEPG === "yes"; + // Inject postgres + if (!this.usingPg) { + return; + } + + this.config.database = { + engine: "postgres", + connectionString: `${process.env.IRCBRIDGE_TEST_PGURL}/${process.env.IRCBRIDGE_TEST_PGDB}`, + }; } /** @@ -40,22 +52,39 @@ class TestEnv { * register the IRC service (as if it were called by app.js). * @return {Promise} which is resolved when the app has finished initiliasing. */ - init(customConfig) { - return this.main.runBridge( - this.config._port, customConfig || this.config, - AppServiceRegistration.fromObject(this.config._registration), true - ).then((ircBridge) => { - console.log("Bridge created"); - this.ircBridge = ircBridge; - }).catch((e) => { - var msg = JSON.stringify(e); + async init(customConfig) { + console.log("FOOO"); + let ircBridge; + try { + if (this.usingPg) { + console.log("FOOO1"); + const db = process.env.IRCBRIDGE_TEST_PGDB; + const superClient = new Client(`${process.env.IRCBRIDGE_TEST_PGURL}/postgres`); + console.log("FOOO2"); + await superClient.connect(); + console.log("FOOO3"); + await superClient.query(`DROP DATABASE IF EXISTS ${db};`); + await superClient.query(`CREATE DATABASE ${db};`); + console.log("FOOO4"); + await superClient.end(); + console.log("FOOO5"); + } + ircBridge = await this.main.runBridge( + this.config._port, customConfig || this.config, + AppServiceRegistration.fromObject(this.config._registration), true + ) + } + catch (ex) { + const msg = JSON.stringify(e); if (e.stack) { msg = e.stack; } console.error("FATAL"); console.error(msg); - }); - + return; + } + console.log("Bridge created"); + this.ircBridge = ircBridge; } /** @@ -96,6 +125,7 @@ module.exports.mkEnv = function() { ); }; + module.exports.initEnv = Promise.coroutine(function*(env, customConfig) { return yield env.init(customConfig); }); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 592f1138f..c563f6fc0 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -319,7 +319,7 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; - const dbConfig = this.config.ircService.database; + const dbConfig = this.config.database; if (dbConfig && dbConfig.engine === "postgres") { this._dataStore = new PgDataStore(this.config.homeserver.domain, dbConfig.connectionString, pkeyPath); yield this._dataStore.ensureSchema(); diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index b051957f1..503850e0b 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -141,4 +141,6 @@ export interface DataStore { removePass(userId: string, domain: string): Promise; getMatrixUserByUsername(domain: string, username: string): Promise; + + destroy(): Promise; } From 0d47c83df295a500615ca2e01b86bc07f96908a6 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 09:28:58 +0100 Subject: [PATCH 031/350] Fixes --- src/datastore/postgres/PgDataStore.ts | 2 +- src/datastore/postgres/schema/v1.ts | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index fbbfd7154..2df5a49ac 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -479,7 +479,7 @@ export class PgDataStore implements DataStore { log.warn("Schema table could not be found"); return 0; } - log.error("Failed to get schema version:", ex); + log.error("Failed to get schema version: %s", ex); } throw Error("Couldn't fetch schema version"); } diff --git a/src/datastore/postgres/schema/v1.ts b/src/datastore/postgres/schema/v1.ts index d58b02419..d3434c062 100644 --- a/src/datastore/postgres/schema/v1.ts +++ b/src/datastore/postgres/schema/v1.ts @@ -60,16 +60,5 @@ export async function runSchema(connection: PoolClient) { count INTEGER ); - CREATE TABLE ( - origin TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - irc_domain TEXT NOT NULL, - irc_channel TEXT NOT NULL, - irc_json JSON NOT NULL, - matrix_json JSON NOT NULL, - CONSTRAINT cons_rooms_unique UNIQUE(room_id, irc_domain, irc_channel) - ); - INSERT INTO ipv6_counter VALUES (0);`); } From 5c90033cb926a07f001f4edb4eff80ee597baff3 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 09:29:14 +0100 Subject: [PATCH 032/350] Docker compose file for postgres tests --- .buildkite/docker-compose.postgres.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .buildkite/docker-compose.postgres.yml diff --git a/.buildkite/docker-compose.postgres.yml b/.buildkite/docker-compose.postgres.yml new file mode 100644 index 000000000..294731686 --- /dev/null +++ b/.buildkite/docker-compose.postgres.yml @@ -0,0 +1,20 @@ +version: '3.1' + +services: + postgres: + image: postgres:11 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: node:12 + depends_on: + - postgres + environment: + IRCBRIDGE_TEST_PGDB: "ircbridge_integtest" + IRCBRIDGE_TEST_PGURL: "postgresql://postgres:postgres@postgres" + IRCBRIDGE_TEST_ENABLEPG: "yes" + working_dir: /app + volumes: + - ..:/app + command: "bash -c 'npm install && npm run build && npm run test'" \ No newline at end of file From 7eeb06f623e69bfef9304bc03952ee0f756daf54 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 17:30:56 +0100 Subject: [PATCH 033/350] Add support for testing the IRC bridge on PostgreSQL --- spec/util/test.js | 62 +++++++++++++++------------ src/bridge/IrcBridge.js | 7 ++- src/bridge/MatrixHandler.js | 1 - src/datastore/NedbDataStore.ts | 4 ++ src/datastore/postgres/PgDataStore.ts | 19 ++++++++ 5 files changed, 63 insertions(+), 30 deletions(-) diff --git a/spec/util/test.js b/spec/util/test.js index df09278ad..393284b54 100644 --- a/spec/util/test.js +++ b/spec/util/test.js @@ -8,6 +8,8 @@ const clientMock = require("./client-sdk-mock"); const ircMock = require("./irc-client-mock"); const { Client } = require('pg'); +const USING_PG = process.env.IRCBRIDGE_TEST_ENABLEPG === "yes"; + clientMock["@global"] = true; ircMock["@global"] = true; const main = proxyquire("../../lib/main.js", { @@ -27,6 +29,20 @@ jasmine.getEnv().addReporter({ } }); +let pgClient; +let pgClientConnectPromise; + +if (USING_PG) { + // Setup postgres for the whole process. + pgClient = new Client(`${process.env.IRCBRIDGE_TEST_PGURL}/postgres`); + pgClientConnectPromise = (async () => { + await pgClient.connect(); + })(); + process.on("beforeExit", async () => { + pgClient.end(); + }) +} + class TestEnv { constructor(config, mockAppService) { this.config = config; @@ -35,15 +51,15 @@ class TestEnv { this.ircBridge = null; this.ircMock = ircMock; this.clientMock = clientMock; - this.usingPg = process.env.IRCBRIDGE_TEST_ENABLEPG === "yes"; // Inject postgres - if (!this.usingPg) { + if (!USING_PG) { return; } + this.pgDb = process.env.IRCBRIDGE_TEST_PGDB; this.config.database = { engine: "postgres", - connectionString: `${process.env.IRCBRIDGE_TEST_PGURL}/${process.env.IRCBRIDGE_TEST_PGDB}`, + connectionString: `${process.env.IRCBRIDGE_TEST_PGURL}/${this.pgDb}`, }; } @@ -53,29 +69,15 @@ class TestEnv { * @return {Promise} which is resolved when the app has finished initiliasing. */ async init(customConfig) { - console.log("FOOO"); let ircBridge; try { - if (this.usingPg) { - console.log("FOOO1"); - const db = process.env.IRCBRIDGE_TEST_PGDB; - const superClient = new Client(`${process.env.IRCBRIDGE_TEST_PGURL}/postgres`); - console.log("FOOO2"); - await superClient.connect(); - console.log("FOOO3"); - await superClient.query(`DROP DATABASE IF EXISTS ${db};`); - await superClient.query(`CREATE DATABASE ${db};`); - console.log("FOOO4"); - await superClient.end(); - console.log("FOOO5"); - } ircBridge = await this.main.runBridge( this.config._port, customConfig || this.config, - AppServiceRegistration.fromObject(this.config._registration), true + AppServiceRegistration.fromObject(this.config._registration), !USING_PG ) } - catch (ex) { - const msg = JSON.stringify(e); + catch (e) { + let msg = JSON.stringify(e); if (e.stack) { msg = e.stack; } @@ -83,7 +85,6 @@ class TestEnv { console.error(msg); return; } - console.log("Bridge created"); this.ircBridge = ircBridge; } @@ -91,15 +92,16 @@ class TestEnv { * Reset the test environment for a new test case that has just run. * This kills the bridge. **/ - afterEach() { + async afterEach() { if (!this.main) { - return Promise.resolve(); + return; } // If there was a previous bridge running, kill it - // This is prevent IRC clients spamming the logs - return this.main.killBridge(this.ircBridge).then(() => { - if (global.gc) {global.gc();} - }) + // This prevents IRC clients spamming the logs + await this.main.killBridge(this.ircBridge); + if (global.gc) { + global.gc(); + } } /** @@ -108,6 +110,12 @@ class TestEnv { async beforeEach() { ircMock._reset(); clientMock._reset(); + if (USING_PG) { + const db = process.env.IRCBRIDGE_TEST_PGDB; + await pgClientConnectPromise; + await pgClient.query(`DROP DATABASE IF EXISTS ${db}`); + await pgClient.query(`CREATE DATABASE ${db}`); + } this.mockAppService = MockAppService.instance(); return true; } diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 1a983af30..8a603c536 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -559,9 +559,12 @@ IrcBridge.prototype._addRequestCallbacks = function() { // usefull once this has been called. // // See (BridgedClient.prototype.kill) -IrcBridge.prototype.kill = function() { +IrcBridge.prototype.kill = async function() { log.info("Killing all clients"); - return this._clientPool.killAllClients(); + await this._clientPool.killAllClients(); + if (this._dataStore) { + await this._dataStore.destroy(); + } } IrcBridge.prototype.isStartedUp = function() { diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index a9ad259b5..3141cc793 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -1482,7 +1482,6 @@ MatrixHandler.prototype._onAliasQuery = Promise.coroutine(function*(req, roomAli let matrixRooms = yield this.ircBridge.getStore().getMatrixRoomsForChannel( channelInfo.server, channelInfo.channel ); - if (matrixRooms.length === 0) { // ====== Track the IRC channel // lower case the name to join (there's a bug in the IRC lib diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 3df73cbfa..61650d6ec 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -602,6 +602,10 @@ export class NeDBDataStore implements DataStore { return matrixUsers[0]; } + public async destroy() { + // This will no-op + } + private static createPmId(userId: string, virtualUserId: string) { // space as delimiter as none of these IDs allow spaces. return "PM_" + userId + " " + virtualUserId; // clobber based on this. diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 63efed973..db8771e2a 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -36,6 +36,7 @@ export class PgDataStore implements DataStore { public static readonly LATEST_SCHEMA = 1; private pgPool: Pool; + private hasEnded: boolean = false; private cryptoStore?: StringCrypto; constructor(private bridgeDomain: string, connectionString: string, pkeyPath?: string, min: number = 1, max: number = 4) { @@ -44,11 +45,17 @@ export class PgDataStore implements DataStore { min, max, }); + this.pgPool.on("error", (err) => { + log.error("Postgres Error: %s", err); + }); if (pkeyPath) { this.cryptoStore = new StringCrypto(); this.cryptoStore.load(pkeyPath); } process.on("beforeExit", (e) => { + if (this.hasEnded) { + return; + } // Ensure we clean up on exit this.pgPool.end(); }) @@ -497,6 +504,18 @@ export class PgDataStore implements DataStore { log.info(`Database schema is at version v${currentVersion}`); } + public async destroy() { + log.info("Destroy called"); + if (this.hasEnded) { + // No-op if end has already been called. + return; + } + this.hasEnded = true; + await this.pgPool.end(); + log.info("PostgresSQL connection ended"); + // This will no-op + } + private async updateSchemaVersion(version: number) { log.debug(`updateSchemaVersion: ${version}`); await this.pgPool.query("UPDATE schema SET version = $1;", [ version ]); From b108f60e4531af2fe4c684002072fa63b7e2b4dd Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 17:32:00 +0100 Subject: [PATCH 034/350] Make postgres pass the tests --- spec/integ/matrix-to-irc.spec.js | 7 ++-- src/bridge/IrcBridge.js | 14 ++++---- src/bridge/RoomAccessSyncer.js | 6 ++-- src/datastore/postgres/PgDataStore.ts | 52 ++++++++++++++++++--------- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/spec/integ/matrix-to-irc.spec.js b/spec/integ/matrix-to-irc.spec.js index 224663a93..a9292fb0f 100644 --- a/spec/integ/matrix-to-irc.spec.js +++ b/spec/integ/matrix-to-irc.spec.js @@ -608,7 +608,7 @@ describe("Matrix-to-Matrix message bridging", function() { yield test.afterEach(env); })); - it("should bridge matrix messages to other mapped matrix rooms", function(done) { + it("should bridge matrix messages to other mapped matrix rooms", test.coroutine(function*() { let testText = "Here is some test text."; let sdk = env.clientMock._client(mirroredUserId); sdk.sendEvent.and.callFake(function(roomId, type, content) { @@ -617,11 +617,10 @@ describe("Matrix-to-Matrix message bridging", function() { body: testText, msgtype: "m.text" }); - done(); return Promise.resolve(); }); - env.mockAppService._trigger("type:m.room.message", { + yield env.mockAppService._trigger("type:m.room.message", { content: { body: testText, msgtype: "m.text" @@ -630,7 +629,7 @@ describe("Matrix-to-Matrix message bridging", function() { room_id: roomMapping.roomId, type: "m.room.message" }); - }); + })); it("should NOT bridge matrix messages to other mapped matrix rooms for PMs", test.coroutine(function*() { diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 8a603c536..e51c1888e 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -86,7 +86,7 @@ function IrcBridge(config, registration) { let bridgeStoreConfig = {}; - if (this.confg.database.engine === "nedb") { + if (this.config.database.engine === "nedb") { const dirPath = this.config.database.connectionString.substring("nedb://".length); bridgeStoreConfig = { roomStore: `${dirPath}/rooms.db`, @@ -335,14 +335,13 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; const dbConfig = this.config.database; - if (dbConfig && dbConfig.engine === "postgres") { + if (dbConfig.engine === "postgres") { + log.info("Using PgDataStore for Datastore"); this._dataStore = new PgDataStore(this.config.homeserver.domain, dbConfig.connectionString, pkeyPath); yield this._dataStore.ensureSchema(); } - else if (dbConfig) { - throw Error("Incorrect database config"); - } - else { + else if (dbConfig.engine === "nedb") { + log.info("Using NeDBDataStore for Datastore"); this._dataStore = new NeDBDataStore( this._bridge.getUserStore(), this._bridge.getRoomStore(), @@ -350,6 +349,9 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { this.config.homeserver.domain, ); } + else { + throw Error("Incorrect database config"); + } yield this._dataStore.removeConfigMappings(); this._identGenerator = new IdentGenerator(this._dataStore); diff --git a/src/bridge/RoomAccessSyncer.js b/src/bridge/RoomAccessSyncer.js index 635a6a3b2..0cbb0e903 100644 --- a/src/bridge/RoomAccessSyncer.js +++ b/src/bridge/RoomAccessSyncer.js @@ -312,9 +312,9 @@ class RoomAccessSyncer { ); } // "k" and "i" - matrixRooms.map((room) => { - this._ircBridge.getStore().setModeForRoom(room.getId(), mode, enabled); - }); + await Promise.all(matrixRooms.map((room) => + this._ircBridge.getStore().setModeForRoom(room.getId(), mode, enabled) + )); const promises = matrixRooms.map((room) => { switch (mode) { diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index db8771e2a..9781b0f71 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -28,6 +28,7 @@ import * as logging from "../../logging"; import Bluebird from "bluebird"; import { stat } from "fs"; import { StringCrypto } from "../StringCrypto"; +import { toIrcLowerCase } from "../../irc/formatting"; const log = logging.get("PgDatastore"); @@ -66,6 +67,7 @@ export class PgDataStore implements DataStore { for (const channel of Object.keys(serverConfig.mappings)) { const ircRoom = new IrcRoom(server, channel); + ircRoom.set("type", "channel"); for (const roomId of serverConfig.mappings[channel]) { const mxRoom = new MatrixRoom(roomId); await this.storeRoom(ircRoom, mxRoom, "config"); @@ -77,17 +79,24 @@ export class PgDataStore implements DataStore { if (typeof origin !== "string") { throw new Error('Origin must be a string = "config"|"provision"|"alias"|"join"'); } - log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s)", - matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin); - const ircRoomSerial = ircRoom.serialize() as any; + log.info("storeRoom (id=%s, addr=%s, chan=%s, origin=%s, type=%s)", + matrixRoom.getId(), ircRoom.getDomain(), ircRoom.channel, origin, ircRoom.getType()); + // We need to *clone* this as we are about to be evil. + const ircRoomSerial = JSON.parse(JSON.stringify(ircRoom.serialize())); + // These keys do not need to be stored inside the JSON blob as we store them + // inside dedicated columns. They will be reinserted into the JSON blob + // when fetched. + const type = ircRoom.getType(); + const domain = ircRoom.getDomain(); + const channel = ircRoom.getChannel(); delete ircRoomSerial.domain; delete ircRoomSerial.channel; delete ircRoomSerial.type; - this.upsertRoom( + await this.upsertRoom( origin, - ircRoom.getType(), - ircRoom.getDomain(), - ircRoom.getChannel(), + type, + domain, + channel, matrixRoom.getId(), JSON.stringify(ircRoomSerial), JSON.stringify(matrixRoom.serialize()), @@ -128,11 +137,13 @@ export class PgDataStore implements DataStore { } public async getRoom(roomId: string, ircDomain: string, ircChannel: string, origin?: RoomOrigin): Promise { - let statement = "SELECT * FROM rooms WHERE room_id = $1, irc_domain = $2, irc_channel = $3"; + let statement = "SELECT * FROM rooms WHERE room_id = $1 AND irc_domain = $2 AND irc_channel = $3"; + let params = [roomId, ircDomain, ircChannel]; if (origin) { - statement += ", origin = $4"; + statement += " AND origin = $4"; + params = params.concat(origin); } - const pgEntry = await this.pgPool.query(statement, [roomId, ircDomain, ircChannel, origin]); + const pgEntry = await this.pgPool.query(statement, params); if (!pgEntry.rowCount) { return null; } @@ -178,7 +189,7 @@ export class PgDataStore implements DataStore { public async removeRoom(roomId: string, ircDomain: string, ircChannel: string, origin: RoomOrigin): Promise { await this.pgPool.query( - "DELETE FROM rooms WHERE room_id = $1, irc_domain = $2, irc_channel = $3, origin = $4", + "DELETE FROM rooms WHERE room_id = $1 AND irc_domain = $2 AND irc_channel = $3 AND origin = $4", [roomId, ircDomain, ircChannel, origin] ); } @@ -222,7 +233,8 @@ export class PgDataStore implements DataStore { const entries = await this.pgPool.query("SELECT room_id, matrix_json FROM rooms WHERE irc_domain = $1 AND irc_channel = $2", [ server.domain, - channel, + // Channels must be lowercase + toIrcLowerCase(channel), ]); return entries.rows.map((e) => new MatrixRoom(e.room_id, e.matrix_json)); } @@ -237,7 +249,8 @@ export class PgDataStore implements DataStore { const entries = await this.pgPool.query(statement, [ server.domain, - channel, + // Channels must be lowercase + toIrcLowerCase(channel), ].concat(origin)); return entries.rows.map((e) => PgDataStore.pgToRoomEntry(e)); } @@ -250,7 +263,8 @@ export class PgDataStore implements DataStore { "WHERE irc_domain = $1 AND irc_channel = $2", [ server.domain, - channel, + // Channels must be lowercase + toIrcLowerCase(channel), ]); entries.rows.forEach((e) => { mapping[e.room_id] = e.modes || []; @@ -281,11 +295,12 @@ export class PgDataStore implements DataStore { } entry.remote.set("modes", modes); - const ircRoomSerial = entry.remote.serialize() as any; + // Clone the object + const ircRoomSerial = JSON.parse(JSON.stringify(entry.remote.serialize())); delete ircRoomSerial.domain; delete ircRoomSerial.channel; delete ircRoomSerial.type; - await this.pgPool.query("UPDATE rooms WHERE room_id = $1, irc_channel = $2, irc_domain = $3 SET irc_json = $4", [ + await this.pgPool.query("UPDATE rooms SET irc_json = $4 WHERE room_id = $1 AND irc_channel = $2 AND irc_domain = $3", [ roomId, entry.remote.get("channel"), entry.remote.get("domain"), @@ -408,6 +423,9 @@ export class PgDataStore implements DataStore { if (!userId) { throw Error("IrcClientConfig does not contain a userId"); } + log.debug(`Storing client configuration for ${userId}`); + // We need to make sure we have a matrix user in the store. + await this.pgPool.query("INSERT INTO matrix_users VALUES ($1, NULL) ON CONFLICT DO NOTHING", [userId]); let password = undefined; if (config.getPassword() && this.cryptoStore) { password = this.cryptoStore.encrypt(config.getPassword()!); @@ -482,6 +500,8 @@ export class PgDataStore implements DataStore { ); if (res.rowCount === 0) { return; + } else if (res.rowCount > 1) { + log.error("getMatrixUserByUsername returned %s results for %s on %s", res.rowCount, username, domain); } return new MatrixUser(res.rows[0].user_id, res.rows[0].data); } From bf19dfd6edc4d271c71a33f6f0f2d2df83ef8b9e Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 23 Sep 2019 17:32:18 +0100 Subject: [PATCH 035/350] Unravel the fabric of hell --- spec/integ/dynamic-channels.spec.js | 1 - spec/integ/provisioning.spec.js | 138 ++++++++++++++++---------- src/datastore/postgres/PgDataStore.ts | 1 - 3 files changed, 83 insertions(+), 57 deletions(-) diff --git a/spec/integ/dynamic-channels.spec.js b/spec/integ/dynamic-channels.spec.js index 1f4e5f2f2..efecd733e 100644 --- a/spec/integ/dynamic-channels.spec.js +++ b/spec/integ/dynamic-channels.spec.js @@ -191,7 +191,6 @@ describe("Dynamic channels", function() { // when we get the create room request, process it. let sdk = env.clientMock._client(config._botUserId); sdk.createRoom.and.callFake(function(opts) { - console.log(opts); expect(opts.room_version).toEqual("the-best-version"); return Promise.resolve({ room_id: tRoomId diff --git a/spec/integ/provisioning.spec.js b/spec/integ/provisioning.spec.js index 958af82ef..926389ee8 100644 --- a/spec/integ/provisioning.spec.js +++ b/spec/integ/provisioning.spec.js @@ -88,7 +88,6 @@ describe("Provisioning API", function() { // Listen for m.room.bridging let sdk = env.clientMock._client(config._botUserId); sdk.sendStateEvent.and.callFake((roomId, kind, content) => { - console.log(roomId, kind, content); if (kind === "m.room.bridging") { if (content.status === "pending") { env.isPending.resolve(); @@ -285,89 +284,120 @@ describe("Provisioning API", function() { describe("link endpoint", function() { - it("should create a M<--->I link", - mockLink({}, true, true)); + // Hello future person. Please do NOT write your tests like this. It is + // very difficult to follow what is going on here and this actually introduced + // a bug where all the tests ran in parallel. For the time being these tests will + // be left in this function soup mess because we know the tests work, but please + // write your tests clearly. - it("should create a M<--->I link for a channel that has capital letters in it", - mockLink({remote_room_channel: '#SomeCaps'}, true, true)); + it("should create a M<--->I link", async () => { + await mockLink({}, true, true); + }); + + + it("should create a M<--->I link for a channel that has capital letters in it", async () => { + await mockLink({remote_room_channel: '#SomeCaps'}, true, true); + }); + - it("should not create a M<--->I link with the same id as one existing", - mockLink({ + it("should not create a M<--->I link with the same id as one existing", async () => { + await mockLink({ matrix_room_id : '!foo:bar', remote_room_server : 'irc.example', - remote_room_channel : '#coffee'}, false, true)); + remote_room_channel : '#coffee'}, false, true); + }); + - it("should not create a M<--->I link when room_id is malformed", - mockLink({matrix_room_id : '!fooooooo'}, false, true)); + it("should not create a M<--->I link when room_id is malformed", async () => { + await mockLink({matrix_room_id : '!fooooooo'}, false, true); + }); - it("should not create a M<--->I link when remote_room_server is malformed", - mockLink({remote_room_server : 'irc./example'}, false, true)); + it("should not create a M<--->I link when remote_room_server is malformed", async () => { + await mockLink({remote_room_server : 'irc./example'}, false, true); + }); - it("should not create a M<--->I link when remote_room_channel is malformed", - mockLink({remote_room_channel : 'coffe####e'}, false, true)); + it("should not create a M<--->I link when remote_room_channel is malformed", async () => { + await mockLink({remote_room_channel : 'coffe####e'}, false, true); + }); // See dynamicChannels.exclude in config file it("should not create a M<--->I link when remote_room_channel is excluded by the " + - "config", - mockLink({remote_room_channel : '#excluded_channel'}, false, true)); + "config", async () => { + await mockLink({remote_room_channel : '#excluded_channel'}, false, true); + }); - it("should not create a M<--->I link when matrix_room_id is not defined", - mockLink({matrix_room_id : null}, false, true)); + it("should not create a M<--->I link when matrix_room_id is not defined", async () => { + await mockLink({matrix_room_id : null}, false, true); + }); - it("should not create a M<--->I link when remote_room_server is not defined", - mockLink({remote_room_server : null}, false, true)); + it("should not create a M<--->I link when remote_room_server is not defined", async () => { + await mockLink({remote_room_server : null}, false, true); + }); - it("should not create a M<--->I link when remote_room_channel is not defined", - mockLink({remote_room_channel : null}, false, true)); + it("should not create a M<--->I link when remote_room_channel is not defined", async () => { + await mockLink({remote_room_channel : null}, false, true); + }); - it("should not create a M<--->I link when op_nick is not defined", - mockLink({op_nick : null}, false, true)); + it("should not create a M<--->I link when op_nick is not defined", async () => { + await mockLink({op_nick : null}, false, true); + }); - it("should not create a M<--->I link when op_nick is not in the room", - mockLink({op_nick : 'somenonexistantop'}, false, true)); + it("should not create a M<--->I link when op_nick is not in the room", async () => { + await mockLink({op_nick : 'somenonexistantop'}, false, true); + }); it("should not create a M<--->I link when op_nick is not an operator, but is in the " + - "room", - mockLink({op_nick : notOp.nick}, false, true)); - - it("should not create a M<--->I link when user does not have enough power in room", - mockLink({user_id: 'powerless'}, false, true)); + "room", async () => { + await mockLink({op_nick : notOp.nick}, false, true); + }); + it("should not create a M<--->I link when user does not have enough power in room", async () => { + await mockLink({user_id: 'powerless'}, false, true); + }); }); describe("unlink endpoint", function() { - it("should remove an existing M<--->I link", - mockLink({}, true, false)); + it("should remove an existing M<--->I link", async () => { + await mockLink({}, true, false) + }); - it("should not remove a non-existing M<--->I link", - mockLink({matrix_room_id : '!idonot:exist'}, false, false, false)); + it("should not remove a non-existing M<--->I link", async () => { + await mockLink({matrix_room_id : '!idonot:exist'}, false, false, false) + }); - it("should not remove a non-provision M<--->I link", - mockLink({ + it("should not remove a non-provision M<--->I link", async () => { + await mockLink({ matrix_room_id : '!foo:bar', remote_room_server : 'irc.example', - remote_room_channel : '#coffee'}, false, false)); + remote_room_channel : '#coffee'}, false, false) + }); - it("should not remove a M<--->I link when room_id is malformed", - mockLink({matrix_room_id : '!fooooooooo'}, false, false)); + it("should not remove a M<--->I link when room_id is malformed", async () => { + await mockLink({matrix_room_id : '!fooooooooo'}, false, false) + }); - it("should not remove a M<--->I link when remote_room_server is malformed", - mockLink({remote_room_server : 'irc./example'}, false, false)); + it("should not remove a M<--->I link when remote_room_server is malformed", async () => { + await mockLink({remote_room_server : 'irc./example'}, false, false) + }); - it("should not remove a M<--->I link when remote_room_channel is malformed", - mockLink({remote_room_channel : 'coffe####e'}, false, false)); + it("should not remove a M<--->I link when remote_room_channel is malformed", async () => { + await mockLink({remote_room_channel : 'coffe####e'}, false, false) + }); it("should not remove a M<--->I link when matrix_room_id is " + - "not defined", - mockLink({matrix_room_id : null}, false, true)); + "not defined", async () => { + await mockLink({matrix_room_id : null}, false, true) + }); it("should not remove a M<--->I link when remote_room_server is " + - "not defined", - mockLink({remote_room_server : null}, false, true)); + "not defined", async () => { + await mockLink({remote_room_server : null}, false, true) + }); it("should not remove a M<--->I link when remote_room_channel is " + - "not defined", - mockLink({remote_room_channel : null}, false, true)); + "not defined", async () => { + await mockLink({remote_room_channel : null}, false, true) + }); }); }); @@ -437,8 +467,6 @@ describe("Provisioning API", function() { let sdk = env.clientMock._client(config._botUserId); sdk.sendStateEvent.and.callFake((roomId, kind, content) => { // Status of m.room.bridging is a success - // console.log(roomId, kind, content); - console.log(roomId, kind, content); if (kind === "m.room.bridging") { if (content.status === "pending") { env.isPending.resolve(); @@ -463,9 +491,9 @@ describe("Provisioning API", function() { yield test.afterEach(env); })); - it("should not create a M<--->I link of the same link id", - mockLink({}, false, true) - ); + it("should not create a M<--->I link of the same link id", async () => { + await mockLink({}, false, true) + }); }); describe("message sending and joining", function() { diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 9781b0f71..888866364 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -245,7 +245,6 @@ export class PgDataStore implements DataStore { } const inStatement = origin.map((_, i) => `\$${i + 3}`).join(", "); const statement = `SELECT * FROM rooms WHERE irc_domain = $1 AND irc_channel = $2 AND origin IN (${inStatement})`; - console.log(statement); const entries = await this.pgPool.query(statement, [ server.domain, From 437886032520cff4cc5a5272bc79d707a201f7fb Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 24 Sep 2019 00:28:55 +0100 Subject: [PATCH 036/350] Spacing --- spec/integ/provisioning.spec.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/spec/integ/provisioning.spec.js b/spec/integ/provisioning.spec.js index 926389ee8..089c10c41 100644 --- a/spec/integ/provisioning.spec.js +++ b/spec/integ/provisioning.spec.js @@ -293,12 +293,10 @@ describe("Provisioning API", function() { it("should create a M<--->I link", async () => { await mockLink({}, true, true); }); - it("should create a M<--->I link for a channel that has capital letters in it", async () => { await mockLink({remote_room_channel: '#SomeCaps'}, true, true); }); - it("should not create a M<--->I link with the same id as one existing", async () => { await mockLink({ @@ -306,7 +304,6 @@ describe("Provisioning API", function() { remote_room_server : 'irc.example', remote_room_channel : '#coffee'}, false, true); }); - it("should not create a M<--->I link when room_id is malformed", async () => { await mockLink({matrix_room_id : '!fooooooo'}, false, true); From 230ecd50ae41c91cdd6b25b2effdd3ee5cc25d69 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 24 Sep 2019 16:38:06 +0100 Subject: [PATCH 037/350] Do not permit var in TS files --- .ts.eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.ts.eslintrc b/.ts.eslintrc index 74fd521d8..c859487f5 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -5,5 +5,6 @@ "rules": { "@typescript-eslint/ban-ts-ignore": 0, "@typescript-eslint/explicit-function-return-type": 0, + "no-var": 2, } } From 14a67481b12141d1391979466418363cd129a260 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 24 Sep 2019 16:43:19 +0100 Subject: [PATCH 038/350] Complete port of ClientPool to typescript --- src/bridge/IrcBridge.js | 2 +- src/irc/ClientPool.ts | 693 ++++++++++++++++++++-------------------- 2 files changed, 348 insertions(+), 347 deletions(-) diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 910e5b796..2cc5228ad 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -10,7 +10,7 @@ var MemberListSyncer = require("./MemberListSyncer.js"); var IdentGenerator = require("../irc/IdentGenerator.js"); var Ipv6Generator = require("../irc/Ipv6Generator.js"); const { IrcServer } = require("../irc/IrcServer.js"); -var ClientPool = require("../irc/ClientPool"); +const { ClientPool } = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); var IrcUser = require("../models/IrcUser"); diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 1dcd79af9..1982e4bdc 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -16,9 +16,11 @@ limitations under the License. const stats = require("../config/stats"); const log = require("../logging").get("ClientPool"); -const Promise = require("bluebird"); const QueuePool = require("../util/QueuePool"); +import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; +import { IrcClientConfig } from "../models/IrcClientConfig"; +import { IrcServer } from "../irc/IrcServer"; /* * Maintains a lookup of connected IRC clients. These connections are transient @@ -49,7 +51,7 @@ export class ClientPool { this.reconnectQueues = { }; } - public nickIsVirtual(server: any, nick: string) { + public nickIsVirtual(server: IrcServer, nick: string): boolean { if (!this.virtualClients[server.domain]) { return false; } @@ -59,23 +61,23 @@ export class ClientPool { } // The client may not have signalled to us that it's connected, but it is connect*ing*. - const pending = Object.keys(this._virtualClients[server.domain].pending || {}); + const pending = Object.keys(this.virtualClients[server.domain].pending || {}); return pending.includes(nick); } - public killAllClients() { + public killAllClients(): Bluebird { const domainList = Object.keys(this.virtualClients); let clients: any[] = []; domainList.forEach((domain) => { clients = clients.concat( Object.keys(this.virtualClients[domain].nicks).map( - (nick) => this.virtualClients[domain].nicks[nick] + (nick: string) => this.virtualClients[domain].nicks[nick] ) ); clients = clients.concat( Object.keys(this.virtualClients[domain].userIds).map( - (userId) => this.virtualClients[domain].userIds[userId] + (userId: string) => this.virtualClients[domain].userIds[userId] ) ); @@ -84,398 +86,397 @@ export class ClientPool { clients = clients.filter((c) => Boolean(c)); - return Promise.all( + return Bluebird.all( clients.map( (client) => client.kill() ) ); } -} -ClientPool.prototype.getOrCreateReconnectQueue = function(server) { - if (server.getConcurrentReconnectLimit() === 0) { - return null; - } - let q = this._reconnectQueues[server.domain]; - if (q === undefined) { - q = this._reconnectQueues[server.domain] = new QueuePool( - server.getConcurrentReconnectLimit(), - (item) => { - log.info(`Reconnecting client. ${q.waitingItems} left.`); - return this._reconnectClient(item); - } - ); - } - return q; -}; - -ClientPool.prototype.setBot = function(server, client) { - this._botClients[server.domain] = client; -}; - -ClientPool.prototype.getBot = function(server) { - return this._botClients[server.domain]; -}; - -ClientPool.prototype.createIrcClient = function(ircClientConfig, matrixUser, isBot) { - var bridgedClient = this._ircBridge.createBridgedClient( - ircClientConfig, matrixUser, isBot - ); - var server = bridgedClient.server; - - if (this._virtualClients[server.domain] === undefined) { - this._virtualClients[server.domain] = { - nicks: Object.create(null), - userIds: Object.create(null), - pending: {}, - }; - this._virtualClientCounts[server.domain] = 0; - } - if (isBot) { - this._botClients[server.domain] = bridgedClient; + public getOrCreateReconnectQueue(server: IrcServer) { + if (server.getConcurrentReconnectLimit() === 0) { + return null; + } + let q = this.reconnectQueues[server.domain]; + if (q === undefined) { + q = this.reconnectQueues[server.domain] = new QueuePool( + server.getConcurrentReconnectLimit(), + (item: any) => { + log.info(`Reconnecting client. ${q.waitingItems} left.`); + return this.reconnectClient(item); + } + ); + } + return q; } - // `pending` is used to ensure that we know if a nick belongs to a userId - // before they have been connected. It's impossible to know for sure - // what nick they will be assigned before being connected, but this - // should catch most cases. Knowing the nick is important, because - // slow clients may not send a 'client-connected' signal before a join is - // emitted, which means ghost users may join with their nickname into matrix. - this._virtualClients[server.domain].pending[bridgedClient.nick] = bridgedClient.userId; - - // add event listeners - bridgedClient.on("client-connected", this._onClientConnected.bind(this)); - bridgedClient.on("client-disconnected", this._onClientDisconnected.bind(this)); - bridgedClient.on("nick-change", this._onNickChange.bind(this)); - bridgedClient.on("join-error", this._onJoinError.bind(this)); - bridgedClient.on("irc-names", this._onNames.bind(this)); - - // store the bridged client immediately in the pool even though it isn't - // connected yet, else we could spawn 2 clients for a single user if this - // function is called quickly. - this._virtualClients[server.domain].userIds[bridgedClient.userId] = bridgedClient; - this._virtualClientCounts[server.domain] = this._virtualClientCounts[server.domain] + 1; - - // Does this server have a max clients limit? If so, check if the limit is - // reached and start cycling based on oldest time. - this._checkClientLimit(server); - return bridgedClient; -}; - -ClientPool.prototype.getBridgedClientByUserId = function(server, userId) { - if (!this._virtualClients[server.domain]) { - return undefined; - } - var cli = this._virtualClients[server.domain].userIds[userId]; - if (!cli || cli.isDead()) { - return undefined; + + public setBot(server: IrcServer, client: any) { + this.botClients[server.domain] = client; } - return cli; -}; -ClientPool.prototype.getBridgedClientByNick = function(server, nick) { - var bot = this.getBot(server); - if (bot && bot.nick === nick) { - return bot; + public getBot(server: IrcServer) { + return this.botClients[server.domain]; } - if (!this._virtualClients[server.domain]) { - return undefined; + public createIrcClient(ircClientConfig: IrcClientConfig, matrixUser: any, isBot: boolean) { + const bridgedClient = this.ircBridge.createBridgedClient( + ircClientConfig, matrixUser, isBot + ); + const server = bridgedClient.server; + + if (this.virtualClients[server.domain] === undefined) { + this.virtualClients[server.domain] = { + nicks: Object.create(null), + userIds: Object.create(null), + pending: {}, + }; + this.virtualClientCounts[server.domain] = 0; + } + if (isBot) { + this.botClients[server.domain] = bridgedClient; + } + + // `pending` is used to ensure that we know if a nick belongs to a userId + // before they have been connected. It's impossible to know for sure + // what nick they will be assigned before being connected, but this + // should catch most cases. Knowing the nick is important, because + // slow clients may not send a 'client-connected' signal before a join is + // emitted, which means ghost users may join with their nickname into matrix. + this.virtualClients[server.domain].pending[bridgedClient.nick] = bridgedClient.userId; + + // add event listeners + bridgedClient.on("client-connected", this.onClientConnected.bind(this)); + bridgedClient.on("client-disconnected", this.onClientDisconnected.bind(this)); + bridgedClient.on("nick-change", this.onNickChange.bind(this)); + bridgedClient.on("join-error", this.onJoinError.bind(this)); + bridgedClient.on("irc-names", this.onNames.bind(this)); + + // store the bridged client immediately in the pool even though it isn't + // connected yet, else we could spawn 2 clients for a single user if this + // function is called quickly. + this.virtualClients[server.domain].userIds[bridgedClient.userId] = bridgedClient; + this.virtualClientCounts[server.domain] = this.virtualClientCounts[server.domain] + 1; + + // Does this server have a max clients limit? If so, check if the limit is + // reached and start cycling based on oldest time. + this.checkClientLimit(server); + return bridgedClient; } - var cli = this._virtualClients[server.domain].nicks[nick]; - if (!cli || cli.isDead()) { - return undefined; + + public getBridgedClientByUserId(server: IrcServer, userId: string) { + if (!this.virtualClients[server.domain]) { + return undefined; + } + const cli = this.virtualClients[server.domain].userIds[userId]; + if (!cli || cli.isDead()) { + return undefined; + } + return cli; } - return cli; -}; - -ClientPool.prototype.getBridgedClientsForUserId = function(userId) { - var domainList = Object.keys(this._virtualClients); - var clientList = []; - domainList.forEach((domain) => { - var cli = this._virtualClients[domain].userIds[userId]; - if (cli && !cli.isDead()) { - clientList.push(cli); + + public getBridgedClientByNick(server: IrcServer, nick: string) { + const bot = this.getBot(server); + if (bot && bot.nick === nick) { + return bot; } - }); - return clientList; -}; - -ClientPool.prototype.getBridgedClientsForRegex = function(userIdRegex) { - userIdRegex = new RegExp(userIdRegex); - const domainList = Object.keys(this._virtualClients); - const clientList = {}; - domainList.forEach((domain) => { - Object.keys( - this._virtualClients[domain].userIds - ).filter( - (u) => userIdRegex.exec(u) !== null - ).forEach((userId) => { - if (!clientList[userId]) { - clientList[userId] = []; + + if (!this.virtualClients[server.domain]) { + return undefined; + } + const cli = this.virtualClients[server.domain].nicks[nick]; + if (!cli || cli.isDead()) { + return undefined; + } + return cli; + } + + public getBridgedClientsForUserId(userId: string) { + const domainList = Object.keys(this.virtualClients); + const clientList: any[] = []; + domainList.forEach((domain) => { + const cli = this.virtualClients[domain].userIds[userId]; + if (cli && !cli.isDead()) { + clientList.push(cli); } - clientList[userId].push(this._virtualClients[domain].userIds[userId]); }); - }); - return clientList; -}; - + return clientList; + } -ClientPool.prototype._checkClientLimit = function(server) { - if (server.getMaxClients() === 0) { - return; + public getBridgedClientsForRegex(userIdRegexString: string) { + const userIdRegex = new RegExp(userIdRegexString); + const domainList = Object.keys(this.virtualClients); + const clientList: {[userId: string]: any} = {}; + domainList.forEach((domain) => { + Object.keys( + this.virtualClients[domain].userIds + ).filter( + (u) => userIdRegex.exec(u) !== null + ).forEach((userId: string) => { + if (!clientList[userId]) { + clientList[userId] = []; + } + clientList[userId].push(this.virtualClients[domain].userIds[userId]); + }); + }); + return clientList; } - var numConnections = this._getNumberOfConnections(server); - this._sendConnectionMetric(server); - if (numConnections < server.getMaxClients()) { - // under the limit, we're good for now. + private checkClientLimit(server: IrcServer) { + if (server.getMaxClients() === 0) { + return; + } + + const numConnections = this.getNumberOfConnections(server); + this.sendConnectionMetric(server); + + if (numConnections < server.getMaxClients()) { + // under the limit, we're good for now. + log.debug( + "%s active connections on %s", + numConnections, server.domain + ); + return; + } + log.debug( - "%s active connections on %s", - numConnections, server.domain + "%s active connections on %s (limit %s)", + numConnections, server.domain, server.getMaxClients() ); - return; - } - log.debug( - "%s active connections on %s (limit %s)", - numConnections, server.domain, server.getMaxClients() - ); - - // find the oldest client to kill. - var oldest = null; - Object.keys(this._virtualClients[server.domain].nicks).forEach((nick) => { - var client = this._virtualClients[server.domain].nicks[nick]; - if (!client) { - // possible since undefined/null values can be present from culled entries + // find the oldest client to kill. + let oldest: any = null; + Object.keys(this.virtualClients[server.domain].nicks).forEach((nick: string) => { + const client = this.virtualClients[server.domain].nicks[nick]; + if (!client) { + // possible since undefined/null values can be present from culled entries + return; + } + if (client.isBot) { + return; // don't ever kick the bot off. + } + if (oldest === null) { + oldest = client; + return; + } + if (client.getLastActionTs() < oldest.getLastActionTs()) { + oldest = client; + } + }); + if (!oldest) { return; } - if (client.isBot) { - return; // don't ever kick the bot off. + // disconnect and remove mappings. + this.removeBridgedClient(oldest); + oldest.disconnect("Client limit exceeded: " + server.getMaxClients()).then( + function() { + log.info("Client limit exceeded: Disconnected %s on %s.", + oldest.nick, oldest.server.domain); + }, + function(e: Error) { + log.error("Error when disconnecting %s on server %s: %s", + oldest.nick, oldest.server.domain, JSON.stringify(e)); + }); + } + + public countTotalConnections(): number { + let count = 0; + + Object.keys(this.virtualClients).forEach((domain) => { + let server = this.ircBridge.getServer(domain); + count += this.getNumberOfConnections(server); + }); + + return count; + } + + public totalReconnectsWaiting (serverDomain: string): number { + if (this.reconnectQueues[serverDomain] !== undefined) { + return this.reconnectQueues[serverDomain].waitingItems; } - if (oldest === null) { - oldest = client; + return 0; + } + + public updateActiveConnectionMetrics(serverDomain: string, ageCounter: any): void { + if (this.virtualClients[serverDomain] === undefined) { return; } - if (client.getLastActionTs() < oldest.getLastActionTs()) { - oldest = client; - } - }); - if (!oldest) { - return; + const clients = Object.values(this.virtualClients[serverDomain].userIds); + clients.forEach((bridgedClient) => { + if (!bridgedClient || bridgedClient.isDead()) { + // We don't want to include dead ones, or ones that don't exist. + return; + } + ageCounter.bump((Date.now() - bridgedClient.getLastActionTs()) / 1000); + }); + } + + public getNickUserIdMappingForChannel(server: IrcServer, channel: string): {[nick: string]: string} { + const nickUserIdMap: {[nick: string]: string} = {}; + const cliSet = this.virtualClients[server.domain].userIds; + Object.keys(cliSet).filter((userId: string) => + cliSet[userId] && cliSet[userId].chanList + && cliSet[userId].chanList.includes(channel) + ).forEach((userId: string) => { + nickUserIdMap[cliSet[userId].nick] = userId; + }); + // Correctly map the bot too. + nickUserIdMap[server.getBotNickname()] = this.ircBridge.getAppServiceUserId(); + return nickUserIdMap; } - // disconnect and remove mappings. - this._removeBridgedClient(oldest); - oldest.disconnect("Client limit exceeded: " + server.getMaxClients()).then( - function() { - log.info("Client limit exceeded: Disconnected %s on %s.", - oldest.nick, oldest.server.domain); - }, - function(e) { - log.error("Error when disconnecting %s on server %s: %s", - oldest.nick, oldest.server.domain, JSON.stringify(e)); - }); -}; - -ClientPool.prototype._getNumberOfConnections = function(server) { - if (!server || !this._virtualClients[server.domain]) { return 0; } - return this._virtualClientCounts[server.domain]; -}; - -ClientPool.prototype.countTotalConnections = function() { - var count = 0; - - Object.keys(this._virtualClients).forEach((domain) => { - let server = this._ircBridge.getServer(domain); - count += this._getNumberOfConnections(server); - }); - - return count; -}; - -ClientPool.prototype.totalReconnectsWaiting = function (serverDomain) { - if (this._reconnectQueues[serverDomain] !== undefined) { - return this._reconnectQueues[serverDomain].waitingItems; + + private getNumberOfConnections(server: IrcServer): number { + if (!server || !this.virtualClients[server.domain]) { return 0; } + return this.virtualClientCounts[server.domain]; } - return 0; -}; -ClientPool.prototype.updateActiveConnectionMetrics = function(server, ageCounter) { - if (this._virtualClients[server] === undefined) { - return; + private sendConnectionMetric(server: IrcServer): void { + stats.ircClients(server.domain, this.getNumberOfConnections(server)); } - const clients = Object.values(this._virtualClients[server].userIds); - clients.forEach((bridgedClient) => { - if (!bridgedClient || bridgedClient.isDead()) { - // We don't want to include dead ones, or ones that don't exist. - return; + + private removeBridgedClient(bridgedClient: any): void { + const server = bridgedClient.server; + this.virtualClients[server.domain].userIds[bridgedClient.userId] = undefined; + this.virtualClients[server.domain].nicks[bridgedClient.nick] = undefined; + this.virtualClientCounts[server.domain] = this.virtualClientCounts[server.domain] - 1; + + if (bridgedClient.isBot) { + this.botClients[server.domain] = undefined; } - ageCounter.bump((Date.now() - bridgedClient.getLastActionTs()) / 1000); - }); -}; - -ClientPool.prototype.getNickUserIdMappingForChannel = function(server, channel) { - const nickUserIdMap = {}; - const cliSet = this._virtualClients[server.domain].userIds; - Object.keys(cliSet).filter((userId) => - cliSet[userId] && cliSet[userId].chanList - && cliSet[userId].chanList.includes(channel) - ).forEach((userId) => { - nickUserIdMap[cliSet[userId].nick] = userId; - }); - // Correctly map the bot too. - nickUserIdMap[server.getBotNickname()] = this._ircBridge.getAppServiceUserId(); - return nickUserIdMap; -}; - -ClientPool.prototype._sendConnectionMetric = function(server) { - stats.ircClients(server.domain, this._getNumberOfConnections(server)); -}; - -ClientPool.prototype._removeBridgedClient = function(bridgedClient) { - var server = bridgedClient.server; - this._virtualClients[server.domain].userIds[bridgedClient.userId] = undefined; - this._virtualClients[server.domain].nicks[bridgedClient.nick] = undefined; - this._virtualClientCounts[server.domain] = this._virtualClientCounts[server.domain] - 1; - - if (bridgedClient.isBot) { - this._botClients[server.domain] = undefined; } -}; -ClientPool.prototype._onClientConnected = function(bridgedClient) { - var server = bridgedClient.server; - var oldNick = bridgedClient.nick; - var actualNick = bridgedClient.unsafeClient.nick; + private onClientConnected(bridgedClient: any): void { + const server = bridgedClient.server; + const oldNick = bridgedClient.nick; + const actualNick = bridgedClient.unsafeClient.nick; - // remove the pending nick we had set for this user - delete this._virtualClients[server.domain].pending[oldNick]; + // remove the pending nick we had set for this user + delete this.virtualClients[server.domain].pending[oldNick]; - // assign a nick to this client - this._virtualClients[server.domain].nicks[actualNick] = bridgedClient; + // assign a nick to this client + this.virtualClients[server.domain].nicks[actualNick] = bridgedClient; - // informative logging - if (oldNick !== actualNick) { - log.debug("Connected with nick '%s' instead of desired nick '%s'", - actualNick, oldNick); + // informative logging + if (oldNick !== actualNick) { + log.debug("Connected with nick '%s' instead of desired nick '%s'", + actualNick, oldNick); + } } -}; -ClientPool.prototype._onClientDisconnected = function(bridgedClient) { - this._removeBridgedClient(bridgedClient); - this._sendConnectionMetric(bridgedClient.server); + private onClientDisconnected(bridgedClient: any): void { + this.removeBridgedClient(bridgedClient); + this.sendConnectionMetric(bridgedClient.server); + + // remove the pending nick we had set for this user + if (this.virtualClients[bridgedClient.server]) { + delete this.virtualClients[bridgedClient.server].pending[bridgedClient.nick]; + } + + if (bridgedClient.disconnectReason === "banned") { + const req = new BridgeRequest(this.ircBridge._bridge.getRequestFactory().newRequest()); + this.ircBridge.matrixHandler.quitUser( + req, bridgedClient.userId, [bridgedClient], + null, "User was banned from the network" + ); + } + + if (bridgedClient.explicitDisconnect) { + // don't reconnect users which explicitly disconnected e.g. client + // cycling, idle timeouts, leaving rooms, etc. + return; + } + // Reconnect this user + // change the client config to use the current nick rather than the desired nick. This + // makes sure that the client attempts to reconnect with the *SAME* nick, and also draws + // from the latest !nick change, as the client config here may be very very old. + const cliConfig = bridgedClient.getClientConfig(); + cliConfig.setDesiredNick(bridgedClient.nick); - // remove the pending nick we had set for this user - if (this._virtualClients[bridgedClient.server]) { - delete this._virtualClients[bridgedClient.server].pending[bridgedClient.nick]; - } - if (bridgedClient.disconnectReason === "banned") { - const req = new BridgeRequest(this._ircBridge._bridge.getRequestFactory().newRequest()); - this._ircBridge.matrixHandler.quitUser( - req, bridgedClient.userId, [bridgedClient], - null, "User was banned from the network" + const cli = this.createIrcClient( + cliConfig, bridgedClient.matrixUser, bridgedClient.isBot ); - } + const chanList = bridgedClient.chanList; + // remove ref to the disconnected client so it can be GC'd. If we don't do this, + // the timeout below holds it in a closure, preventing it from being GC'd. + bridgedClient = undefined; - if (bridgedClient.explicitDisconnect) { - // don't reconnect users which explicitly disconnected e.g. client - // cycling, idle timeouts, leaving rooms, etc. - return; - } - // Reconnect this user - // change the client config to use the current nick rather than the desired nick. This - // makes sure that the client attempts to reconnect with the *SAME* nick, and also draws - // from the latest !nick change, as the client config here may be very very old. - var cliConfig = bridgedClient.getClientConfig(); - cliConfig.setDesiredNick(bridgedClient.nick); - - - var cli = this.createIrcClient( - cliConfig, bridgedClient.matrixUser, bridgedClient.isBot - ); - var chanList = bridgedClient.chanList; - // remove ref to the disconnected client so it can be GC'd. If we don't do this, - // the timeout below holds it in a closure, preventing it from being GC'd. - bridgedClient = undefined; - - if (chanList.length === 0) { - log.info(`Dropping ${cli._id} ${cli.nick} because they are not joined to any channels`); - return; - } - let queue = this.getOrCreateReconnectQueue(cli.server); - if (queue === null) { - this._reconnectClient({ + if (chanList.length === 0) { + log.info(`Dropping ${cli._id} ${cli.nick} because they are not joined to any channels`); + return; + } + let queue = this.getOrCreateReconnectQueue(cli.server); + if (queue === null) { + this.reconnectClient({ + cli: cli, + chanList: chanList, + }); + return; + } + queue.enqueue(cli._id, { cli: cli, chanList: chanList, }); - return; } - queue.enqueue(cli._id, { - cli: cli, - chanList: chanList, - }); -}; - -ClientPool.prototype._reconnectClient = function(cliChan) { - const cli = cliChan.cli; - const chanList = cliChan.chanList; - return cli.connect().then(() => { - log.info( - "<%s> Reconnected %s@%s", cli._id, cli.nick, cli.server.domain - ); - log.info("<%s> Rejoining %s channels", cli._id, chanList.length); - chanList.forEach(function(c) { - cli.joinChannel(c); + + private reconnectClient(cliChan: any): void { + const cli = cliChan.cli; + const chanList: string[] = cliChan.chanList; + return cli.connect().then(() => { + log.info( + "<%s> Reconnected %s@%s", cli._id, cli.nick, cli.server.domain + ); + log.info("<%s> Rejoining %s channels", cli._id, chanList.length); + chanList.forEach(function(c: string) { + cli.joinChannel(c); + }); + this.sendConnectionMetric(cli.server); + }, (e: Error) => { + log.error( + "<%s> Failed to reconnect %s@%s", cli._id, cli.nick, cli.server.domain + ); }); - this._sendConnectionMetric(cli.server); - }, (e) => { - log.error( - "<%s> Failed to reconnect %s@%s", cli._id, cli.nick, cli.server.domain - ); - }); -} - -ClientPool.prototype._onNickChange = function(bridgedClient, oldNick, newNick) { - this._virtualClients[bridgedClient.server.domain].nicks[oldNick] = undefined; - this._virtualClients[bridgedClient.server.domain].nicks[newNick] = bridgedClient; -}; - -ClientPool.prototype._onJoinError = Promise.coroutine(function*(bridgedClient, chan, err) { - var errorsThatShouldKick = [ - "err_bannedfromchan", // they aren't allowed in channels they are banned on. - "err_inviteonlychan", // they aren't allowed in invite only channels - "err_channelisfull", // they aren't allowed in if the channel is full - "err_badchannelkey", // they aren't allowed in channels with a bad key - "err_needreggednick", // they aren't allowed in +r channels if they haven't authed - ]; - if (errorsThatShouldKick.indexOf(err) === -1) { - return; } - if (!bridgedClient.userId || bridgedClient.isBot) { - return; // the bot itself can get these join errors + + private onNickChange(bridgedClient: any, oldNick: string, newNick: string): void { + this.virtualClients[bridgedClient.server.domain].nicks[oldNick] = undefined; + this.virtualClients[bridgedClient.server.domain].nicks[newNick] = bridgedClient; } - // TODO: this is a bit evil, no one in their right mind would expect - // the client pool to be kicking matrix users from a room :( - log.info(`Kicking ${bridgedClient.userId} from room due to ${err}`); - let matrixRooms = yield this._ircBridge.getStore().getMatrixRoomsForChannel( - bridgedClient.server, chan - ); - let promises = matrixRooms.map((room) => { - return this._ircBridge.getAppServiceBridge().getIntent().kick( - room.getId(), bridgedClient.userId, `IRC error on ${chan}: ${err}` + + private async onJoinError (bridgedClient: any, chan: string, err: string): Promise { + const errorsThatShouldKick = [ + "err_bannedfromchan", // they aren't allowed in channels they are banned on. + "err_inviteonlychan", // they aren't allowed in invite only channels + "err_channelisfull", // they aren't allowed in if the channel is full + "err_badchannelkey", // they aren't allowed in channels with a bad key + "err_needreggednick", // they aren't allowed in +r channels if they haven't authed + ]; + if (!errorsThatShouldKick.includes(err)) { + return; + } + if (!bridgedClient.userId || bridgedClient.isBot) { + return; // the bot itself can get these join errors + } + // TODO: this is a bit evil, no one in their right mind would expect + // the client pool to be kicking matrix users from a room :( + log.info(`Kicking ${bridgedClient.userId} from room due to ${err}`); + let matrixRooms = await this.ircBridge.getStore().getMatrixRoomsForChannel( + bridgedClient.server, chan ); - }); - yield Promise.all(promises); -}); - -ClientPool.prototype._onNames = Promise.coroutine(function*(bridgedClient, chan, names) { - let mls = this._ircBridge.memberListSyncers[bridgedClient.server.domain]; - if (!mls) { - return; + let promises = matrixRooms.map((room: any) => { + return this.ircBridge.getAppServiceBridge().getIntent().kick( + room.getId(), bridgedClient.userId, `IRC error on ${chan}: ${err}` + ); + }); + await Promise.all(promises); } - yield mls.updateIrcMemberList(chan, names); -}); -module.exports = ClientPool; + private onNames(bridgedClient: any, chan: string, names: any): Bluebird { + let mls = this.ircBridge.memberListSyncers[bridgedClient.server.domain]; + if (!mls) { + return Bluebird.resolve(); + } + return mls.updateIrcMemberList(chan, names); + } +} \ No newline at end of file From 3fd65b7ad27636ab01d64b1155ac4a444d255a93 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 17 Sep 2019 13:15:44 +0100 Subject: [PATCH 039/350] Convert IrcServer to Typescript --- spec/unit/IrcServer.spec.js | 2 +- src/bridge/IrcBridge.js | 2 +- src/irc/IrcServer.js | 582 ------------------------------- src/irc/IrcServer.ts | 672 ++++++++++++++++++++++++++++++++++++ src/main.js | 2 +- 5 files changed, 675 insertions(+), 585 deletions(-) delete mode 100644 src/irc/IrcServer.js create mode 100644 src/irc/IrcServer.ts diff --git a/spec/unit/IrcServer.spec.js b/spec/unit/IrcServer.spec.js index ab359b3ae..33b04befa 100644 --- a/spec/unit/IrcServer.spec.js +++ b/spec/unit/IrcServer.spec.js @@ -1,5 +1,5 @@ "use strict"; -const IrcServer = require("../../lib/irc/IrcServer"); +const { IrcServer } = require("../../lib/irc/IrcServer"); const extend = require("extend"); describe("IrcServer", function() { describe("getNick", function() { diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index b960fd21d..43d6e411c 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -9,7 +9,7 @@ var MatrixHandler = require("./MatrixHandler.js"); var MemberListSyncer = require("./MemberListSyncer.js"); var IdentGenerator = require("../irc/IdentGenerator.js"); var Ipv6Generator = require("../irc/Ipv6Generator.js"); -var IrcServer = require("../irc/IrcServer.js"); +const { IrcServer } = require("../irc/IrcServer.js"); var ClientPool = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); diff --git a/src/irc/IrcServer.js b/src/irc/IrcServer.js deleted file mode 100644 index e06bece02..000000000 --- a/src/irc/IrcServer.js +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Represents a single IRC server from config.yaml - */ -"use strict"; -const logging = require("../logging"); -const { IrcClientConfig } = require("../models/IrcClientConfig"); -const log = logging.get("IrcServer"); -const BridgedClient = require("./BridgedClient"); - -const GROUP_ID_REGEX = /^\+\S+:\S+$/; - -/** - * Construct a new IRC Server. - * @constructor - * @param {string} domain : The IRC network address - * @param {Object} serverConfig : The config options for this network. - * @param {string} homeserverDomain : The domain of the homeserver - * e.g "matrix.org" - * @param {Number} expiryTimeSeconds : How old a matrix message can be - * before it is considered 'expired' and not sent to IRC. If 0, messages - * will never expire. - */ -function IrcServer(domain, serverConfig, homeserverDomain, expiryTimeSeconds) { - this.domain = domain; - this.config = serverConfig; - - this._addresses = serverConfig.additionalAddresses; - if (!this._addresses) { - this._addresses = []; - } - this._addresses.push(domain); - this._homeserverDomain = homeserverDomain; - this._expiryTimeSeconds = expiryTimeSeconds; - - if (this.config.dynamicChannels.groupId !== undefined && - this.config.dynamicChannels.groupId.trim() !== "") { - this._groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; - if (!this._groupIdValid) { - log.warn( -`${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` - ); - } - } - else { - this._groupIdValid = false; - } -} - -/** - * Get how old a matrix message can be (in seconds) before it is considered - * 'expired' and not sent to IRC. - * @return {Number} The number of seconds. If 0, they never expire. - */ -IrcServer.prototype.getExpiryTimeSeconds = function() { - return this._expiryTimeSeconds || 0; -} - -/** - * Get a string that represents the human-readable name for a server. - * @return {string} this.config.name if truthy, otherwise it will return - * an empty string. - */ -IrcServer.prototype.getReadableName = function() { - let name = this.config.name; - if (name) { - return name; - } - - return ''; -} - -/** - * Return a randomised server domain from the default and additional addresses. - * @return {string} - */ -IrcServer.prototype.randomDomain = function() { - return this._addresses[ - Math.floor((Math.random() * 1000) % this._addresses.length) - ]; -} - -/** - * Returns the network ID of this server, which should be unique across all - * IrcServers on the bridge. Defaults to the domain of this IrcServer. - * @return {string} this.config.networkId || this.domain - */ -IrcServer.prototype.getNetworkId = function() { - return this.config.networkId || this.domain; -} - -/** - * Returns whether the server is configured to wait getQuitDebounceDelayMs before - * parting a user that has disconnected due to a net-split. - * @return {Boolean} this.config.quitDebounce.enabled. - */ -IrcServer.prototype.shouldDebounceQuits = function() { - return this.config.quitDebounce.enabled; -} - -/** - * Get the minimum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If the user rejoins a channel before bridging - * the quit to a leave, the leave will not be sent. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMinMs = function() { - return this.config.quitDebounce.delayMinMs; -} - -/** - * Get the maximum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If a leave is bridged, it will occur at a - * random time between delayMinMs (see above) delayMaxMs. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMaxMs = function() { - return this.config.quitDebounce.delayMaxMs; -} - -/** - * Get the rate of maximum quits received per second before a net-split is - * detected. If the rate of quits received becomes higher that this value, - * a net split is considered ongoing. - * @return {number} - */ -IrcServer.prototype.getDebounceQuitsPerSecond = function() { - return this.config.quitDebounce.quitsPerSecond; -} - -/** - * Get a map that converts IRC user modes to Matrix power levels. - * @return {Object} - */ -IrcServer.prototype.getModePowerMap = function() { - return this.config.modePowerMap || {}; -} - -IrcServer.prototype.getHardCodedRoomIds = function() { - var roomIds = new Set(); - var channels = Object.keys(this.config.mappings); - channels.forEach((chan) => { - this.config.mappings[chan].forEach((roomId) => { - roomIds.add(roomId); - }); - }); - return Array.from(roomIds.keys()); -}; - -IrcServer.prototype.shouldSendConnectionNotices = function() { - return this.config.sendConnectionMessages; -}; - -IrcServer.prototype.isBotEnabled = function() { - return this.config.botConfig.enabled; -}; - -IrcServer.prototype.getUserModes = function() { - return this.config.ircClients.userModes || ""; -} - -IrcServer.prototype.getJoinRule = function() { - return this.config.dynamicChannels.joinRule; -}; - -IrcServer.prototype.areGroupsEnabled = function() { - return this._groupIdValid; -}; - -IrcServer.prototype.getGroupId = function() { - return this.config.dynamicChannels.groupId; -}; - -IrcServer.prototype.shouldFederatePMs = function() { - return this.config.privateMessages.federate; -}; - -IrcServer.prototype.getMemberListFloodDelayMs = function() { - return this.config.membershipLists.floodDelayMs; -}; - -IrcServer.prototype.shouldFederate = function() { - return this.config.dynamicChannels.federate; -}; -IrcServer.prototype.forceRoomVersion = function() { - return this.config.dynamicChannels.roomVersion; -}; - -IrcServer.prototype.getPort = function() { - return this.config.port; -}; - -IrcServer.prototype.isInWhitelist = function(userId) { - return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; -}; - -IrcServer.prototype.getCA = function() { - return this.config.ca; -}; - -IrcServer.prototype.useSsl = function() { - return Boolean(this.config.ssl); -}; - -IrcServer.prototype.useSslSelfSigned = function() { - return Boolean(this.config.sslselfsign); -}; - -IrcServer.prototype.useSasl = function() { - return Boolean(this.config.sasl); -}; - -IrcServer.prototype.allowExpiredCerts = function() { - return Boolean(this.config.allowExpiredCerts); -}; - -IrcServer.prototype.getIdleTimeout = function() { - return this.config.ircClients.idleTimeout; -}; - -IrcServer.prototype.getReconnectIntervalMs = function() { - return this.config.ircClients.reconnectIntervalMs; -}; - -IrcServer.prototype.getConcurrentReconnectLimit = function() { - return this.config.ircClients.concurrentReconnectLimit; -}; - -IrcServer.prototype.getMaxClients = function() { - return this.config.ircClients.maxClients; -}; - -IrcServer.prototype.shouldPublishRooms = function() { - return this.config.dynamicChannels.published; -}; - -IrcServer.prototype.allowsNickChanges = function() { - return this.config.ircClients.allowNickChanges; -}; - -IrcServer.prototype.getBotNickname = function() { - return this.config.botConfig.nick; -}; - -IrcServer.prototype.createBotIrcClientConfig = function(username) { - return IrcClientConfig.newConfig( - null, this.domain, this.config.botConfig.nick, username, - this.config.botConfig.password - ); -}; - -IrcServer.prototype.getIpv6Prefix = function() { - return this.config.ircClients.ipv6.prefix; -}; - -IrcServer.prototype.getIpv6Only = function() { - return this.config.ircClients.ipv6.only; -}; - -IrcServer.prototype.getLineLimit = function() { - return this.config.ircClients.lineLimit; -}; - -IrcServer.prototype.getJoinAttempts = function() { - return this.config.matrixClients.joinAttempts; -}; - -IrcServer.prototype.isExcludedChannel = function(channel) { - return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; -}; - -IrcServer.prototype.hasInviteRooms = function() { - return ( - this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" - ); -}; - -// check if this server dynamically create rooms with aliases. -IrcServer.prototype.createsDynamicAliases = function() { - return ( - this.config.dynamicChannels.enabled && - this.config.dynamicChannels.createAlias - ); -}; - -// check if this server dynamically creates rooms which are joinable via an alias only. -IrcServer.prototype.createsPublicAliases = function() { - return ( - this.createsDynamicAliases() && - this.getJoinRule() === "public" - ); -}; - -IrcServer.prototype.allowsPms = function() { - return this.config.privateMessages.enabled; -}; - -IrcServer.prototype.shouldSyncMembershipToIrc = function(kind, roomId) { - return this._shouldSyncMembership(kind, roomId, true); -}; - -IrcServer.prototype.shouldSyncMembershipToMatrix = function(kind, channel) { - return this._shouldSyncMembership(kind, channel, false); -}; - -IrcServer.prototype._shouldSyncMembership = function(kind, identifier, toIrc) { - if (["incremental", "initial"].indexOf(kind) === -1) { - throw new Error("Bad kind: " + kind); - } - if (!this.config.membershipLists.enabled) { - return false; - } - var shouldSync = this.config.membershipLists.global[ - toIrc ? "matrixToIrc" : "ircToMatrix" - ][kind]; - - if (!identifier) { - return shouldSync; - } - - // check for specific rules for the room id / channel - if (toIrc) { - // room rules clobber global rules - this.config.membershipLists.rooms.forEach(function(r) { - if (r.room === identifier && r.matrixToIrc) { - shouldSync = r.matrixToIrc[kind]; - } - }); - } - else { - // channel rules clobber global rules - this.config.membershipLists.channels.forEach(function(chan) { - if (chan.channel === identifier && chan.ircToMatrix) { - shouldSync = chan.ircToMatrix[kind]; - } - }); - } - - return shouldSync; -}; - -IrcServer.prototype.shouldJoinChannelsIfNoUsers = function() { - return this.config.botConfig.joinChannelsIfNoUsers; -}; - -IrcServer.prototype.isMembershipListsEnabled = function() { - return this.config.membershipLists.enabled; -}; - -IrcServer.prototype.getUserLocalpart = function(nick) { - // the template is just a literal string with special vars; so find/replace - // the vars and strip the @ - var uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); - return uid.replace(/\$NICK/g, nick).substring(1); -}; - -IrcServer.prototype.claimsUserId = function(userId) { - // the server claims the given user ID if the ID matches the user ID template. - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(userId); -}; - -IrcServer.prototype.getNickFromUserId = function(userId) { - // extract the nick from the given user ID - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*?)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(userId); - if (!match) { - return null; - } - return match[1]; -}; - -IrcServer.prototype.getUserIdFromNick = function(nick) { - var template = this.config.matrixClients.userTemplate; - return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + - ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getDisplayNameFromNick = function(nick) { - var template = this.config.matrixClients.displayName; - var displayName = template.replace(/\$NICK/g, nick); - displayName = displayName.replace(/\$SERVER/g, this.domain); - return displayName; -}; - -IrcServer.prototype.claimsAlias = function(alias) { - // the server claims the given alias if the alias matches the alias template - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "#(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(alias); -}; - -IrcServer.prototype.getChannelFromAlias = function(alias) { - // extract the channel from the given alias - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "([^:]*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(alias); - if (!match) { - return null; - } - log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); - return match[1]; -}; - -IrcServer.prototype.getAliasFromChannel = function(channel) { - var template = this.config.dynamicChannels.aliasTemplate; - return template.replace(/\$CHANNEL/, channel) + ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getNick = function(userId, displayName) { - const illegalChars = BridgedClient.illegalCharactersRegex; - let localpart = userId.substring(1).split(":")[0]; - localpart = localpart.replace(illegalChars, ""); - displayName = displayName ? displayName.replace(illegalChars, "") : undefined; - const display = [displayName, localpart].find((n) => Boolean(n)); - if (!display) { - throw new Error("Could not get nick for user, all characters were invalid"); - } - const template = this.config.ircClients.nickTemplate; - let nick = template.replace(/\$USERID/g, userId); - nick = nick.replace(/\$LOCALPART/g, localpart); - nick = nick.replace(/\$DISPLAY/g, display); - return nick; -}; - -IrcServer.prototype.getAliasRegex = function() { - return templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -IrcServer.prototype.getUserRegex = function() { - return templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$NICK": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -function templateToRegex(template, literalVars, regexVars, suffix) { - // The 'template' is a literal string with some special variables which need - // to be find/replaced. - var regex = template; - Object.keys(literalVars).forEach(function(varPlaceholder) { - regex = regex.replace( - new RegExp(escapeRegExp(varPlaceholder), 'g'), - literalVars[varPlaceholder] - ); - }); - - // at this point the template is still a literal string, so escape it before - // applying the regex vars. - regex = escapeRegExp(regex); - // apply regex vars - Object.keys(regexVars).forEach(function(varPlaceholder) { - regex = regex.replace( - // double escape, because we bluntly escaped the entire string before - // so our match is now escaped. - new RegExp(escapeRegExp(escapeRegExp(varPlaceholder)), 'g'), - regexVars[varPlaceholder] - ); - }); - - suffix = suffix || ""; - return regex + suffix; -} - -function escapeRegExp(string) { - // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions - return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -IrcServer.DEFAULT_CONFIG = { - sendConnectionMessages: true, - quitDebounce: { - enabled: false, - quitsPerSecond: 5, - delayMinMs: 3600000, // 1h - delayMaxMs: 7200000, // 2h - }, - botConfig: { - nick: "appservicebot", - joinChannelsIfNoUsers: true, - enabled: true - }, - privateMessages: { - enabled: true, - exclude: [], - federate: true - }, - dynamicChannels: { - enabled: false, - published: true, - createAlias: true, - joinRule: "public", - federate: true, - aliasTemplate: "#irc_$SERVER_$CHANNEL", - whitelist: [], - exclude: [] - }, - mappings: {}, - matrixClients: { - userTemplate: "@$SERVER_$NICK", - displayName: "$NICK (IRC)", - joinAttempts: -1, - }, - ircClients: { - nickTemplate: "M-$DISPLAY", - maxClients: 30, - idleTimeout: 172800, - reconnectIntervalMs: 5000, - concurrentReconnectLimit: 50, - allowNickChanges: false, - ipv6: {only: false}, - lineLimit: 3 - }, - membershipLists: { - enabled: false, - floodDelayMs: 10000, // 10s - global: { - ircToMatrix: { - initial: false, - incremental: false - }, - matrixToIrc: { - initial: false, - incremental: false - } - }, - channels: [], - rooms: [] - } -}; - -module.exports = IrcServer; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts new file mode 100644 index 000000000..40cf1ef92 --- /dev/null +++ b/src/irc/IrcServer.ts @@ -0,0 +1,672 @@ + +import * as logging from "../logging"; +import * as BridgedClient from "./BridgedClient"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +const log = logging.get("IrcServer"); +const GROUP_ID_REGEX = /^\+\S+:\S+$/ + +type MembershipSyncKind = "incremental"|"initial"; + +/* + * Represents a single IRC server from config.yaml + */ +export class IrcServer { + private addresses: string[]; + private groupIdValid: boolean; + /** + * Construct a new IRC Server. + * @constructor + * @param {string} domain : The IRC network address + * @param {Object} serverConfig : The config options for this network. + * @param {string} homeserverDomain : The domain of the homeserver + * e.g "matrix.org" + * @param {number} expiryTimeSeconds : How old a matrix message can be + * before it is considered 'expired' and not sent to IRC. If 0, messages + * will never expire. + */ + constructor(public domain: string, public config: IrcServerConfig, + private homeserverDomain: string, private expiryTimeSeconds: number = 0) { + this.addresses = config.additionalAddresses || []; + this.addresses.push(domain); + + if (this.config.dynamicChannels.groupId !== undefined && + this.config.dynamicChannels.groupId.trim() !== "") { + this.groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; + if (!this.groupIdValid) { + log.warn( + `${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` + ); + } + } + else { + this.groupIdValid = false; + } + } + + /** + * Get how old a matrix message can be (in seconds) before it is considered + * 'expired' and not sent to IRC. + * @return {Number} The number of seconds. If 0, they never expire. + */ + public getExpiryTimeSeconds() { + return this.expiryTimeSeconds; + } + + /** + * Get a string that represents the human-readable name for a server. + * @return {string} this.config.name if truthy, otherwise it will return + * an empty string. + */ + public getReadableName() { + return this.config.name || ""; + } + + /** + * Return a randomised server domain from the default and additional addresses. + * @return {string} + */ + public randomDomain() { + return this.addresses[ + Math.floor((Math.random() * 1000) % this.addresses.length) + ]; + } + + /** + * Returns the network ID of this server, which should be unique across all + * IrcServers on the bridge. Defaults to the domain of this IrcServer. + * @return {string} this.config.networkId || this.domain + */ + public getNetworkId() { + return this.config.networkId || this.domain; + } + + /** + * Returns whether the server is configured to wait getQuitDebounceDelayMs before + * parting a user that has disconnected due to a net-split. + * @return {Boolean} this.config.quitDebounce.enabled. + */ + public shouldDebounceQuits() { + return this.config.quitDebounce.enabled; + } + + /** + * Get the minimum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If the user rejoins a channel before bridging + * the quit to a leave, the leave will not be sent. + * @return {number} + */ + public getQuitDebounceDelayMinMs() { + return this.config.quitDebounce.delayMinMs; + } + + /** + * Get the maximum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If a leave is bridged, it will occur at a + * random time between delayMinMs (see above) delayMaxMs. + * @return {number} + */ + public getQuitDebounceDelayMaxMs() { + return this.config.quitDebounce.delayMaxMs; + } + + /** + * Get the rate of maximum quits received per second before a net-split is + * detected. If the rate of quits received becomes higher that this value, + * a net split is considered ongoing. + * @return {number} + */ + public getDebounceQuitsPerSecond() { + return this.config.quitDebounce.quitsPerSecond; + } + + /** + * Get a map that converts IRC user modes to Matrix power levels. + * @return {Object} + */ + public getModePowerMap() { + return this.config.modePowerMap || {}; + } + + public getHardCodedRoomIds() { + const roomIds = new Set(); + const channels = Object.keys(this.config.mappings); + channels.forEach((chan) => { + this.config.mappings[chan].forEach((roomId) => { + roomIds.add(roomId); + }); + }); + return Array.from(roomIds.keys()); + } + + public shouldSendConnectionNotices() { + return this.config.sendConnectionMessages; + } + + public isBotEnabled() { + return this.config.botConfig.enabled; + } + + public getUserModes() { + return this.config.ircClients.userModes || ""; + } + + public getJoinRule() { + return this.config.dynamicChannels.joinRule; + } + + public areGroupsEnabled() { + return this.groupIdValid; + } + + public getGroupId() { + return this.config.dynamicChannels.groupId; + } + + public shouldFederatePMs() { + return this.config.privateMessages.federate; + } + + public getMemberListFloodDelayMs() { + return this.config.membershipLists.floodDelayMs; + } + + public shouldFederate() { + return this.config.dynamicChannels.federate; + } + public forceRoomVersion() { + return this.config.dynamicChannels.roomVersion; + } + + public getPort() { + return this.config.port; + } + + public isInWhitelist(userId: string) { + return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; + } + + public getCA() { + return this.config.ca; + } + + public useSsl() { + return Boolean(this.config.ssl); + } + + public useSslSelfSigned() { + return Boolean(this.config.sslselfsign); + } + + public useSasl() { + return Boolean(this.config.sasl); + } + + public allowExpiredCerts() { + return Boolean(this.config.allowExpiredCerts); + } + + public getIdleTimeout() { + return this.config.ircClients.idleTimeout; + } + + public getReconnectIntervalMs() { + return this.config.ircClients.reconnectIntervalMs; + } + + public getConcurrentReconnectLimit() { + return this.config.ircClients.concurrentReconnectLimit; + } + + public getMaxClients() { + return this.config.ircClients.maxClients; + } + + public shouldPublishRooms() { + return this.config.dynamicChannels.published; + } + + public allowsNickChanges() { + return this.config.ircClients.allowNickChanges; + } + + public getBotNickname() { + return this.config.botConfig.nick; + } + + public createBotIrcClientConfig(username: string) { + return IrcClientConfig.newConfig( + null, this.domain, this.config.botConfig.nick, username, + this.config.botConfig.password + ); + } + + public getIpv6Prefix() { + return this.config.ircClients.ipv6.prefix; + } + + public getIpv6Only() { + return this.config.ircClients.ipv6.only; + } + + public getLineLimit() { + return this.config.ircClients.lineLimit; + } + + public getJoinAttempts() { + return this.config.matrixClients.joinAttempts; + } + + public isExcludedChannel(channel: string) { + return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; + } + + public hasInviteRooms() { + return ( + this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" + ); + } + + // check if this server dynamically create rooms with aliases. + public createsDynamicAliases() { + return ( + this.config.dynamicChannels.enabled && + this.config.dynamicChannels.createAlias + ); + } + + // check if this server dynamically creates rooms which are joinable via an alias only. + public createsPublicAliases() { + return ( + this.createsDynamicAliases() && + this.getJoinRule() === "public" + ); + } + + public allowsPms() { + return this.config.privateMessages.enabled; + } + + public shouldSyncMembershipToIrc(kind: MembershipSyncKind, roomId: string) { + return this._shouldSyncMembership(kind, roomId, true); + } + + public shouldSyncMembershipToMatrix(kind: MembershipSyncKind, channel: string) { + return this._shouldSyncMembership(kind, channel, false); + } + + public _shouldSyncMembership(kind: MembershipSyncKind, identifier: string, toIrc: boolean) { + if (["incremental", "initial"].indexOf(kind) === -1) { + throw new Error("Bad kind: " + kind); + } + if (!this.config.membershipLists.enabled) { + return false; + } + let shouldSync = this.config.membershipLists.global[ + toIrc ? "matrixToIrc" : "ircToMatrix" + ][kind]; + + if (!identifier) { + return shouldSync; + } + + // check for specific rules for the room id / channel + if (toIrc) { + // room rules clobber global rules + this.config.membershipLists.rooms.forEach(function(r) { + if (r.room === identifier && r.matrixToIrc) { + shouldSync = r.matrixToIrc[kind]; + } + }); + } + else { + // channel rules clobber global rules + this.config.membershipLists.channels.forEach(function(chan) { + if (chan.channel === identifier && chan.ircToMatrix) { + shouldSync = chan.ircToMatrix[kind]; + } + }); + } + + return shouldSync; + } + + public shouldJoinChannelsIfNoUsers() { + return this.config.botConfig.joinChannelsIfNoUsers; + } + + public isMembershipListsEnabled() { + return this.config.membershipLists.enabled; + } + + public getUserLocalpart(nick: string) { + // the template is just a literal string with special vars; so find/replace + // the vars and strip the @ + const uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); + return uid.replace(/\$NICK/g, nick).substring(1); + } + + public claimsUserId(userId: string) { + // the server claims the given user ID if the ID matches the user ID template. + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(userId); + } + + public getNickFromUserId(userId: string) { + // extract the nick from the given user ID + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*?)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(userId); + if (!match) { + return null; + } + return match[1]; + } + + public getUserIdFromNick(nick: string) { + const template = this.config.matrixClients.userTemplate; + return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + + ":" + this.homeserverDomain; + } + + public getDisplayNameFromNick(nick: string) { + const template = this.config.matrixClients.displayName; + let displayName = template.replace(/\$NICK/g, nick); + displayName = displayName.replace(/\$SERVER/g, this.domain); + return displayName; + } + + public claimsAlias(alias: string) { + // the server claims the given alias if the alias matches the alias template + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "#(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(alias); + } + + public getChannelFromAlias(alias: string) { + // extract the channel from the given alias + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "([^:]*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(alias); + if (!match) { + return null; + } + log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); + return match[1]; + } + + public getAliasFromChannel(channel: string) { + const template = this.config.dynamicChannels.aliasTemplate; + return template.replace(/\$CHANNEL/, channel) + ":" + this.homeserverDomain; + } + + public getNick(userId: string, displayName?: string) { + const illegalChars = BridgedClient.illegalCharactersRegex; + let localpart = userId.substring(1).split(":")[0]; + localpart = localpart.replace(illegalChars, ""); + displayName = displayName ? displayName.replace(illegalChars, "") : undefined; + const display = [displayName, localpart].find((n) => Boolean(n)); + if (!display) { + throw new Error("Could not get nick for user, all characters were invalid"); + } + const template = this.config.ircClients.nickTemplate; + let nick = template.replace(/\$USERID/g, userId); + nick = nick.replace(/\$LOCALPART/g, localpart); + nick = nick.replace(/\$DISPLAY/g, display); + return nick; + } + + public getAliasRegex() { + return IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public getUserRegex() { + return IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$NICK": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public static get DEFAULT_CONFIG(): IrcServerConfig { + return { + sendConnectionMessages: true, + quitDebounce: { + enabled: false, + quitsPerSecond: 5, + delayMinMs: 3600000, // 1h + delayMaxMs: 7200000, // 2h + }, + botConfig: { + nick: "appservicebot", + joinChannelsIfNoUsers: true, + enabled: true + }, + privateMessages: { + enabled: true, + exclude: [], + federate: true + }, + dynamicChannels: { + enabled: false, + published: true, + createAlias: true, + joinRule: "public", + federate: true, + aliasTemplate: "#irc_$SERVER_$CHANNEL", + whitelist: [], + exclude: [] + }, + mappings: {}, + matrixClients: { + userTemplate: "@$SERVER_$NICK", + displayName: "$NICK (IRC)", + joinAttempts: -1, + }, + ircClients: { + nickTemplate: "M-$DISPLAY", + maxClients: 30, + idleTimeout: 172800, + reconnectIntervalMs: 5000, + concurrentReconnectLimit: 50, + allowNickChanges: false, + ipv6: { + only: false + }, + lineLimit: 3 + }, + membershipLists: { + enabled: false, + floodDelayMs: 10000, // 10s + global: { + ircToMatrix: { + initial: false, + incremental: false + }, + matrixToIrc: { + initial: false, + incremental: false + } + }, + channels: [], + rooms: [] + } + } + } + + private static templateToRegex(template: string, literalVars: {[key: string]: string}, + regexVars: {[key: string]: string}, suffix: string) { + // The 'template' is a literal string with some special variables which need + // to be find/replaced. + let regex = template; + Object.keys(literalVars).forEach(function(varPlaceholder) { + regex = regex.replace( + new RegExp(IrcServer.escapeRegExp(varPlaceholder), 'g'), + literalVars[varPlaceholder] + ); + }); + + // at this point the template is still a literal string, so escape it before + // applying the regex vars. + regex = IrcServer.escapeRegExp(regex); + // apply regex vars + Object.keys(regexVars).forEach(function(varPlaceholder) { + regex = regex.replace( + // double escape, because we bluntly escaped the entire string before + // so our match is now escaped. + new RegExp(IrcServer.escapeRegExp(IrcServer.escapeRegExp(varPlaceholder)), 'g'), + regexVars[varPlaceholder] + ); + }); + + suffix = suffix || ""; + return regex + suffix; + } + + private static escapeRegExp(s: string) { + // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } +} + +export interface IrcServerConfig { + // These are determined to be always defined or possibly undefined + // by the existence of the keys in IrcServer.DEFAULT_CONFIG. + name?: string; + port?: number; + ca?: string; + networkId?: string; + ssl?: boolean; + sslselfsign?: boolean; + sasl?: boolean; + allowExpiredCerts?: boolean; + additionalAddresses?: string[]; + dynamicChannels: { + enabled: boolean; + published: boolean; + createAlias: boolean; + joinRule: "public"|"invite"; + federate: boolean; + aliasTemplate: string; + whitelist: string[]; + exclude: string[]; + roomVersion?: string; + groupId?: string; + }; + quitDebounce: { + enabled: boolean; + quitsPerSecond: number; + delayMinMs: number; + delayMaxMs: number; + }; + mappings: {[channel: string]: string[]}; // chan -> roomId[] + modePowerMap?: {[mode: string]: number}; + sendConnectionMessages: boolean; + botConfig: { + nick: string; + joinChannelsIfNoUsers: boolean; + enabled: boolean; + password?: string; + }; + privateMessages: { + enabled: boolean; + exclude: string[]; + federate: boolean; + }; + matrixClients: { + userTemplate: string; + displayName: string; + joinAttempts: number; + }; + ircClients: { + nickTemplate: string; + maxClients: number; + idleTimeout: number; + reconnectIntervalMs: number; + concurrentReconnectLimit: number; + allowNickChanges: boolean; + ipv6: { + only: boolean; + prefix?: string; + }; + lineLimit: number; + userModes?: string; + }; + membershipLists: { + enabled: boolean; + floodDelayMs: number; + global: { + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }; + channels: { + channel: string; + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + }[]; + rooms: { + room: string; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }[]; + }; +} diff --git a/src/main.js b/src/main.js index 1bec2fa5a..0a5669566 100644 --- a/src/main.js +++ b/src/main.js @@ -7,7 +7,7 @@ const RoomBridgeStore = require("matrix-appservice-bridge").RoomBridgeStore; const UserBridgeStore = require("matrix-appservice-bridge").UserBridgeStore; const IrcBridge = require("./bridge/IrcBridge.js"); -const IrcServer = require("./irc/IrcServer.js"); +const { IrcServer } = require("./irc/IrcServer.js"); const stats = require("./config/stats"); const ident = require("./irc/ident"); const logging = require("./logging"); From 4c8c8b819eba3e096d0fe8a031dd5571cface7ce Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Sep 2019 20:37:05 +0100 Subject: [PATCH 040/350] Convert IrcServer to Typescript --- spec/unit/IrcServer.spec.js | 2 +- src/bridge/IrcBridge.js | 15 +- src/irc/IrcServer.js | 582 ------------------------------- src/irc/IrcServer.ts | 672 ++++++++++++++++++++++++++++++++++++ src/main.js | 2 +- src/models/IrcRoom.ts | 11 +- 6 files changed, 689 insertions(+), 595 deletions(-) delete mode 100644 src/irc/IrcServer.js create mode 100644 src/irc/IrcServer.ts diff --git a/spec/unit/IrcServer.spec.js b/spec/unit/IrcServer.spec.js index ab359b3ae..33b04befa 100644 --- a/spec/unit/IrcServer.spec.js +++ b/spec/unit/IrcServer.spec.js @@ -1,5 +1,5 @@ "use strict"; -const IrcServer = require("../../lib/irc/IrcServer"); +const { IrcServer } = require("../../lib/irc/IrcServer"); const extend = require("extend"); describe("IrcServer", function() { describe("getNick", function() { diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index b960fd21d..edcd08e6b 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -9,7 +9,7 @@ var MatrixHandler = require("./MatrixHandler.js"); var MemberListSyncer = require("./MemberListSyncer.js"); var IdentGenerator = require("../irc/IdentGenerator.js"); var Ipv6Generator = require("../irc/Ipv6Generator.js"); -var IrcServer = require("../irc/IrcServer.js"); +const { IrcServer } = require("../irc/IrcServer.js"); var ClientPool = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); @@ -18,7 +18,8 @@ const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); -const { DataStore } = require("../DataStore"); +const { NeDBDataStore } = require("../datastore/NedbDataStore"); +const { PgDataStore } = require("../datastore/postgres/PgDataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, @@ -318,11 +319,13 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; + this._dataStore = new NeDBDataStore( + this._bridge.getUserStore(), + this._bridge.getRoomStore(), + pkeyPath, + this.config.homeserver.domain, + ); - this._dataStore = new DataStore( - this._bridge.getUserStore(), this._bridge.getRoomStore(), pkeyPath, - this.config.homeserver.domain - ); yield this._dataStore.removeConfigMappings(); this._identGenerator = new IdentGenerator(this._dataStore); this._ipv6Generator = new Ipv6Generator(this._dataStore); diff --git a/src/irc/IrcServer.js b/src/irc/IrcServer.js deleted file mode 100644 index e06bece02..000000000 --- a/src/irc/IrcServer.js +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Represents a single IRC server from config.yaml - */ -"use strict"; -const logging = require("../logging"); -const { IrcClientConfig } = require("../models/IrcClientConfig"); -const log = logging.get("IrcServer"); -const BridgedClient = require("./BridgedClient"); - -const GROUP_ID_REGEX = /^\+\S+:\S+$/; - -/** - * Construct a new IRC Server. - * @constructor - * @param {string} domain : The IRC network address - * @param {Object} serverConfig : The config options for this network. - * @param {string} homeserverDomain : The domain of the homeserver - * e.g "matrix.org" - * @param {Number} expiryTimeSeconds : How old a matrix message can be - * before it is considered 'expired' and not sent to IRC. If 0, messages - * will never expire. - */ -function IrcServer(domain, serverConfig, homeserverDomain, expiryTimeSeconds) { - this.domain = domain; - this.config = serverConfig; - - this._addresses = serverConfig.additionalAddresses; - if (!this._addresses) { - this._addresses = []; - } - this._addresses.push(domain); - this._homeserverDomain = homeserverDomain; - this._expiryTimeSeconds = expiryTimeSeconds; - - if (this.config.dynamicChannels.groupId !== undefined && - this.config.dynamicChannels.groupId.trim() !== "") { - this._groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; - if (!this._groupIdValid) { - log.warn( -`${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` - ); - } - } - else { - this._groupIdValid = false; - } -} - -/** - * Get how old a matrix message can be (in seconds) before it is considered - * 'expired' and not sent to IRC. - * @return {Number} The number of seconds. If 0, they never expire. - */ -IrcServer.prototype.getExpiryTimeSeconds = function() { - return this._expiryTimeSeconds || 0; -} - -/** - * Get a string that represents the human-readable name for a server. - * @return {string} this.config.name if truthy, otherwise it will return - * an empty string. - */ -IrcServer.prototype.getReadableName = function() { - let name = this.config.name; - if (name) { - return name; - } - - return ''; -} - -/** - * Return a randomised server domain from the default and additional addresses. - * @return {string} - */ -IrcServer.prototype.randomDomain = function() { - return this._addresses[ - Math.floor((Math.random() * 1000) % this._addresses.length) - ]; -} - -/** - * Returns the network ID of this server, which should be unique across all - * IrcServers on the bridge. Defaults to the domain of this IrcServer. - * @return {string} this.config.networkId || this.domain - */ -IrcServer.prototype.getNetworkId = function() { - return this.config.networkId || this.domain; -} - -/** - * Returns whether the server is configured to wait getQuitDebounceDelayMs before - * parting a user that has disconnected due to a net-split. - * @return {Boolean} this.config.quitDebounce.enabled. - */ -IrcServer.prototype.shouldDebounceQuits = function() { - return this.config.quitDebounce.enabled; -} - -/** - * Get the minimum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If the user rejoins a channel before bridging - * the quit to a leave, the leave will not be sent. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMinMs = function() { - return this.config.quitDebounce.delayMinMs; -} - -/** - * Get the maximum number of ms to debounce before bridging a QUIT to Matrix - * during a detected net-split. If a leave is bridged, it will occur at a - * random time between delayMinMs (see above) delayMaxMs. - * @return {number} - */ -IrcServer.prototype.getQuitDebounceDelayMaxMs = function() { - return this.config.quitDebounce.delayMaxMs; -} - -/** - * Get the rate of maximum quits received per second before a net-split is - * detected. If the rate of quits received becomes higher that this value, - * a net split is considered ongoing. - * @return {number} - */ -IrcServer.prototype.getDebounceQuitsPerSecond = function() { - return this.config.quitDebounce.quitsPerSecond; -} - -/** - * Get a map that converts IRC user modes to Matrix power levels. - * @return {Object} - */ -IrcServer.prototype.getModePowerMap = function() { - return this.config.modePowerMap || {}; -} - -IrcServer.prototype.getHardCodedRoomIds = function() { - var roomIds = new Set(); - var channels = Object.keys(this.config.mappings); - channels.forEach((chan) => { - this.config.mappings[chan].forEach((roomId) => { - roomIds.add(roomId); - }); - }); - return Array.from(roomIds.keys()); -}; - -IrcServer.prototype.shouldSendConnectionNotices = function() { - return this.config.sendConnectionMessages; -}; - -IrcServer.prototype.isBotEnabled = function() { - return this.config.botConfig.enabled; -}; - -IrcServer.prototype.getUserModes = function() { - return this.config.ircClients.userModes || ""; -} - -IrcServer.prototype.getJoinRule = function() { - return this.config.dynamicChannels.joinRule; -}; - -IrcServer.prototype.areGroupsEnabled = function() { - return this._groupIdValid; -}; - -IrcServer.prototype.getGroupId = function() { - return this.config.dynamicChannels.groupId; -}; - -IrcServer.prototype.shouldFederatePMs = function() { - return this.config.privateMessages.federate; -}; - -IrcServer.prototype.getMemberListFloodDelayMs = function() { - return this.config.membershipLists.floodDelayMs; -}; - -IrcServer.prototype.shouldFederate = function() { - return this.config.dynamicChannels.federate; -}; -IrcServer.prototype.forceRoomVersion = function() { - return this.config.dynamicChannels.roomVersion; -}; - -IrcServer.prototype.getPort = function() { - return this.config.port; -}; - -IrcServer.prototype.isInWhitelist = function(userId) { - return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; -}; - -IrcServer.prototype.getCA = function() { - return this.config.ca; -}; - -IrcServer.prototype.useSsl = function() { - return Boolean(this.config.ssl); -}; - -IrcServer.prototype.useSslSelfSigned = function() { - return Boolean(this.config.sslselfsign); -}; - -IrcServer.prototype.useSasl = function() { - return Boolean(this.config.sasl); -}; - -IrcServer.prototype.allowExpiredCerts = function() { - return Boolean(this.config.allowExpiredCerts); -}; - -IrcServer.prototype.getIdleTimeout = function() { - return this.config.ircClients.idleTimeout; -}; - -IrcServer.prototype.getReconnectIntervalMs = function() { - return this.config.ircClients.reconnectIntervalMs; -}; - -IrcServer.prototype.getConcurrentReconnectLimit = function() { - return this.config.ircClients.concurrentReconnectLimit; -}; - -IrcServer.prototype.getMaxClients = function() { - return this.config.ircClients.maxClients; -}; - -IrcServer.prototype.shouldPublishRooms = function() { - return this.config.dynamicChannels.published; -}; - -IrcServer.prototype.allowsNickChanges = function() { - return this.config.ircClients.allowNickChanges; -}; - -IrcServer.prototype.getBotNickname = function() { - return this.config.botConfig.nick; -}; - -IrcServer.prototype.createBotIrcClientConfig = function(username) { - return IrcClientConfig.newConfig( - null, this.domain, this.config.botConfig.nick, username, - this.config.botConfig.password - ); -}; - -IrcServer.prototype.getIpv6Prefix = function() { - return this.config.ircClients.ipv6.prefix; -}; - -IrcServer.prototype.getIpv6Only = function() { - return this.config.ircClients.ipv6.only; -}; - -IrcServer.prototype.getLineLimit = function() { - return this.config.ircClients.lineLimit; -}; - -IrcServer.prototype.getJoinAttempts = function() { - return this.config.matrixClients.joinAttempts; -}; - -IrcServer.prototype.isExcludedChannel = function(channel) { - return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; -}; - -IrcServer.prototype.hasInviteRooms = function() { - return ( - this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" - ); -}; - -// check if this server dynamically create rooms with aliases. -IrcServer.prototype.createsDynamicAliases = function() { - return ( - this.config.dynamicChannels.enabled && - this.config.dynamicChannels.createAlias - ); -}; - -// check if this server dynamically creates rooms which are joinable via an alias only. -IrcServer.prototype.createsPublicAliases = function() { - return ( - this.createsDynamicAliases() && - this.getJoinRule() === "public" - ); -}; - -IrcServer.prototype.allowsPms = function() { - return this.config.privateMessages.enabled; -}; - -IrcServer.prototype.shouldSyncMembershipToIrc = function(kind, roomId) { - return this._shouldSyncMembership(kind, roomId, true); -}; - -IrcServer.prototype.shouldSyncMembershipToMatrix = function(kind, channel) { - return this._shouldSyncMembership(kind, channel, false); -}; - -IrcServer.prototype._shouldSyncMembership = function(kind, identifier, toIrc) { - if (["incremental", "initial"].indexOf(kind) === -1) { - throw new Error("Bad kind: " + kind); - } - if (!this.config.membershipLists.enabled) { - return false; - } - var shouldSync = this.config.membershipLists.global[ - toIrc ? "matrixToIrc" : "ircToMatrix" - ][kind]; - - if (!identifier) { - return shouldSync; - } - - // check for specific rules for the room id / channel - if (toIrc) { - // room rules clobber global rules - this.config.membershipLists.rooms.forEach(function(r) { - if (r.room === identifier && r.matrixToIrc) { - shouldSync = r.matrixToIrc[kind]; - } - }); - } - else { - // channel rules clobber global rules - this.config.membershipLists.channels.forEach(function(chan) { - if (chan.channel === identifier && chan.ircToMatrix) { - shouldSync = chan.ircToMatrix[kind]; - } - }); - } - - return shouldSync; -}; - -IrcServer.prototype.shouldJoinChannelsIfNoUsers = function() { - return this.config.botConfig.joinChannelsIfNoUsers; -}; - -IrcServer.prototype.isMembershipListsEnabled = function() { - return this.config.membershipLists.enabled; -}; - -IrcServer.prototype.getUserLocalpart = function(nick) { - // the template is just a literal string with special vars; so find/replace - // the vars and strip the @ - var uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); - return uid.replace(/\$NICK/g, nick).substring(1); -}; - -IrcServer.prototype.claimsUserId = function(userId) { - // the server claims the given user ID if the ID matches the user ID template. - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(userId); -}; - -IrcServer.prototype.getNickFromUserId = function(userId) { - // extract the nick from the given user ID - var regex = templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain - }, - { - "$NICK": "(.*?)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(userId); - if (!match) { - return null; - } - return match[1]; -}; - -IrcServer.prototype.getUserIdFromNick = function(nick) { - var template = this.config.matrixClients.userTemplate; - return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + - ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getDisplayNameFromNick = function(nick) { - var template = this.config.matrixClients.displayName; - var displayName = template.replace(/\$NICK/g, nick); - displayName = displayName.replace(/\$SERVER/g, this.domain); - return displayName; -}; - -IrcServer.prototype.claimsAlias = function(alias) { - // the server claims the given alias if the alias matches the alias template - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "#(.*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - return new RegExp(regex).test(alias); -}; - -IrcServer.prototype.getChannelFromAlias = function(alias) { - // extract the channel from the given alias - var regex = templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain - }, - { - "$CHANNEL": "([^:]*)" - }, - ":" + escapeRegExp(this._homeserverDomain) - ); - var match = new RegExp(regex).exec(alias); - if (!match) { - return null; - } - log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); - return match[1]; -}; - -IrcServer.prototype.getAliasFromChannel = function(channel) { - var template = this.config.dynamicChannels.aliasTemplate; - return template.replace(/\$CHANNEL/, channel) + ":" + this._homeserverDomain; -}; - -IrcServer.prototype.getNick = function(userId, displayName) { - const illegalChars = BridgedClient.illegalCharactersRegex; - let localpart = userId.substring(1).split(":")[0]; - localpart = localpart.replace(illegalChars, ""); - displayName = displayName ? displayName.replace(illegalChars, "") : undefined; - const display = [displayName, localpart].find((n) => Boolean(n)); - if (!display) { - throw new Error("Could not get nick for user, all characters were invalid"); - } - const template = this.config.ircClients.nickTemplate; - let nick = template.replace(/\$USERID/g, userId); - nick = nick.replace(/\$LOCALPART/g, localpart); - nick = nick.replace(/\$DISPLAY/g, display); - return nick; -}; - -IrcServer.prototype.getAliasRegex = function() { - return templateToRegex( - this.config.dynamicChannels.aliasTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -IrcServer.prototype.getUserRegex = function() { - return templateToRegex( - this.config.matrixClients.userTemplate, - { - "$SERVER": this.domain // find/replace $server - }, - { - "$NICK": ".*" // the nick is unknown, so replace with a wildcard - }, - // Only match the domain of the HS - ":" + escapeRegExp(this._homeserverDomain) - ); -}; - -function templateToRegex(template, literalVars, regexVars, suffix) { - // The 'template' is a literal string with some special variables which need - // to be find/replaced. - var regex = template; - Object.keys(literalVars).forEach(function(varPlaceholder) { - regex = regex.replace( - new RegExp(escapeRegExp(varPlaceholder), 'g'), - literalVars[varPlaceholder] - ); - }); - - // at this point the template is still a literal string, so escape it before - // applying the regex vars. - regex = escapeRegExp(regex); - // apply regex vars - Object.keys(regexVars).forEach(function(varPlaceholder) { - regex = regex.replace( - // double escape, because we bluntly escaped the entire string before - // so our match is now escaped. - new RegExp(escapeRegExp(escapeRegExp(varPlaceholder)), 'g'), - regexVars[varPlaceholder] - ); - }); - - suffix = suffix || ""; - return regex + suffix; -} - -function escapeRegExp(string) { - // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions - return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -IrcServer.DEFAULT_CONFIG = { - sendConnectionMessages: true, - quitDebounce: { - enabled: false, - quitsPerSecond: 5, - delayMinMs: 3600000, // 1h - delayMaxMs: 7200000, // 2h - }, - botConfig: { - nick: "appservicebot", - joinChannelsIfNoUsers: true, - enabled: true - }, - privateMessages: { - enabled: true, - exclude: [], - federate: true - }, - dynamicChannels: { - enabled: false, - published: true, - createAlias: true, - joinRule: "public", - federate: true, - aliasTemplate: "#irc_$SERVER_$CHANNEL", - whitelist: [], - exclude: [] - }, - mappings: {}, - matrixClients: { - userTemplate: "@$SERVER_$NICK", - displayName: "$NICK (IRC)", - joinAttempts: -1, - }, - ircClients: { - nickTemplate: "M-$DISPLAY", - maxClients: 30, - idleTimeout: 172800, - reconnectIntervalMs: 5000, - concurrentReconnectLimit: 50, - allowNickChanges: false, - ipv6: {only: false}, - lineLimit: 3 - }, - membershipLists: { - enabled: false, - floodDelayMs: 10000, // 10s - global: { - ircToMatrix: { - initial: false, - incremental: false - }, - matrixToIrc: { - initial: false, - incremental: false - } - }, - channels: [], - rooms: [] - } -}; - -module.exports = IrcServer; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts new file mode 100644 index 000000000..40cf1ef92 --- /dev/null +++ b/src/irc/IrcServer.ts @@ -0,0 +1,672 @@ + +import * as logging from "../logging"; +import * as BridgedClient from "./BridgedClient"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +const log = logging.get("IrcServer"); +const GROUP_ID_REGEX = /^\+\S+:\S+$/ + +type MembershipSyncKind = "incremental"|"initial"; + +/* + * Represents a single IRC server from config.yaml + */ +export class IrcServer { + private addresses: string[]; + private groupIdValid: boolean; + /** + * Construct a new IRC Server. + * @constructor + * @param {string} domain : The IRC network address + * @param {Object} serverConfig : The config options for this network. + * @param {string} homeserverDomain : The domain of the homeserver + * e.g "matrix.org" + * @param {number} expiryTimeSeconds : How old a matrix message can be + * before it is considered 'expired' and not sent to IRC. If 0, messages + * will never expire. + */ + constructor(public domain: string, public config: IrcServerConfig, + private homeserverDomain: string, private expiryTimeSeconds: number = 0) { + this.addresses = config.additionalAddresses || []; + this.addresses.push(domain); + + if (this.config.dynamicChannels.groupId !== undefined && + this.config.dynamicChannels.groupId.trim() !== "") { + this.groupIdValid = GROUP_ID_REGEX.exec(this.config.dynamicChannels.groupId) !== null; + if (!this.groupIdValid) { + log.warn( + `${domain} has an incorrectly configured groupId for dynamicChannels and will not set groups.` + ); + } + } + else { + this.groupIdValid = false; + } + } + + /** + * Get how old a matrix message can be (in seconds) before it is considered + * 'expired' and not sent to IRC. + * @return {Number} The number of seconds. If 0, they never expire. + */ + public getExpiryTimeSeconds() { + return this.expiryTimeSeconds; + } + + /** + * Get a string that represents the human-readable name for a server. + * @return {string} this.config.name if truthy, otherwise it will return + * an empty string. + */ + public getReadableName() { + return this.config.name || ""; + } + + /** + * Return a randomised server domain from the default and additional addresses. + * @return {string} + */ + public randomDomain() { + return this.addresses[ + Math.floor((Math.random() * 1000) % this.addresses.length) + ]; + } + + /** + * Returns the network ID of this server, which should be unique across all + * IrcServers on the bridge. Defaults to the domain of this IrcServer. + * @return {string} this.config.networkId || this.domain + */ + public getNetworkId() { + return this.config.networkId || this.domain; + } + + /** + * Returns whether the server is configured to wait getQuitDebounceDelayMs before + * parting a user that has disconnected due to a net-split. + * @return {Boolean} this.config.quitDebounce.enabled. + */ + public shouldDebounceQuits() { + return this.config.quitDebounce.enabled; + } + + /** + * Get the minimum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If the user rejoins a channel before bridging + * the quit to a leave, the leave will not be sent. + * @return {number} + */ + public getQuitDebounceDelayMinMs() { + return this.config.quitDebounce.delayMinMs; + } + + /** + * Get the maximum number of ms to debounce before bridging a QUIT to Matrix + * during a detected net-split. If a leave is bridged, it will occur at a + * random time between delayMinMs (see above) delayMaxMs. + * @return {number} + */ + public getQuitDebounceDelayMaxMs() { + return this.config.quitDebounce.delayMaxMs; + } + + /** + * Get the rate of maximum quits received per second before a net-split is + * detected. If the rate of quits received becomes higher that this value, + * a net split is considered ongoing. + * @return {number} + */ + public getDebounceQuitsPerSecond() { + return this.config.quitDebounce.quitsPerSecond; + } + + /** + * Get a map that converts IRC user modes to Matrix power levels. + * @return {Object} + */ + public getModePowerMap() { + return this.config.modePowerMap || {}; + } + + public getHardCodedRoomIds() { + const roomIds = new Set(); + const channels = Object.keys(this.config.mappings); + channels.forEach((chan) => { + this.config.mappings[chan].forEach((roomId) => { + roomIds.add(roomId); + }); + }); + return Array.from(roomIds.keys()); + } + + public shouldSendConnectionNotices() { + return this.config.sendConnectionMessages; + } + + public isBotEnabled() { + return this.config.botConfig.enabled; + } + + public getUserModes() { + return this.config.ircClients.userModes || ""; + } + + public getJoinRule() { + return this.config.dynamicChannels.joinRule; + } + + public areGroupsEnabled() { + return this.groupIdValid; + } + + public getGroupId() { + return this.config.dynamicChannels.groupId; + } + + public shouldFederatePMs() { + return this.config.privateMessages.federate; + } + + public getMemberListFloodDelayMs() { + return this.config.membershipLists.floodDelayMs; + } + + public shouldFederate() { + return this.config.dynamicChannels.federate; + } + public forceRoomVersion() { + return this.config.dynamicChannels.roomVersion; + } + + public getPort() { + return this.config.port; + } + + public isInWhitelist(userId: string) { + return this.config.dynamicChannels.whitelist.indexOf(userId) !== -1; + } + + public getCA() { + return this.config.ca; + } + + public useSsl() { + return Boolean(this.config.ssl); + } + + public useSslSelfSigned() { + return Boolean(this.config.sslselfsign); + } + + public useSasl() { + return Boolean(this.config.sasl); + } + + public allowExpiredCerts() { + return Boolean(this.config.allowExpiredCerts); + } + + public getIdleTimeout() { + return this.config.ircClients.idleTimeout; + } + + public getReconnectIntervalMs() { + return this.config.ircClients.reconnectIntervalMs; + } + + public getConcurrentReconnectLimit() { + return this.config.ircClients.concurrentReconnectLimit; + } + + public getMaxClients() { + return this.config.ircClients.maxClients; + } + + public shouldPublishRooms() { + return this.config.dynamicChannels.published; + } + + public allowsNickChanges() { + return this.config.ircClients.allowNickChanges; + } + + public getBotNickname() { + return this.config.botConfig.nick; + } + + public createBotIrcClientConfig(username: string) { + return IrcClientConfig.newConfig( + null, this.domain, this.config.botConfig.nick, username, + this.config.botConfig.password + ); + } + + public getIpv6Prefix() { + return this.config.ircClients.ipv6.prefix; + } + + public getIpv6Only() { + return this.config.ircClients.ipv6.only; + } + + public getLineLimit() { + return this.config.ircClients.lineLimit; + } + + public getJoinAttempts() { + return this.config.matrixClients.joinAttempts; + } + + public isExcludedChannel(channel: string) { + return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; + } + + public hasInviteRooms() { + return ( + this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" + ); + } + + // check if this server dynamically create rooms with aliases. + public createsDynamicAliases() { + return ( + this.config.dynamicChannels.enabled && + this.config.dynamicChannels.createAlias + ); + } + + // check if this server dynamically creates rooms which are joinable via an alias only. + public createsPublicAliases() { + return ( + this.createsDynamicAliases() && + this.getJoinRule() === "public" + ); + } + + public allowsPms() { + return this.config.privateMessages.enabled; + } + + public shouldSyncMembershipToIrc(kind: MembershipSyncKind, roomId: string) { + return this._shouldSyncMembership(kind, roomId, true); + } + + public shouldSyncMembershipToMatrix(kind: MembershipSyncKind, channel: string) { + return this._shouldSyncMembership(kind, channel, false); + } + + public _shouldSyncMembership(kind: MembershipSyncKind, identifier: string, toIrc: boolean) { + if (["incremental", "initial"].indexOf(kind) === -1) { + throw new Error("Bad kind: " + kind); + } + if (!this.config.membershipLists.enabled) { + return false; + } + let shouldSync = this.config.membershipLists.global[ + toIrc ? "matrixToIrc" : "ircToMatrix" + ][kind]; + + if (!identifier) { + return shouldSync; + } + + // check for specific rules for the room id / channel + if (toIrc) { + // room rules clobber global rules + this.config.membershipLists.rooms.forEach(function(r) { + if (r.room === identifier && r.matrixToIrc) { + shouldSync = r.matrixToIrc[kind]; + } + }); + } + else { + // channel rules clobber global rules + this.config.membershipLists.channels.forEach(function(chan) { + if (chan.channel === identifier && chan.ircToMatrix) { + shouldSync = chan.ircToMatrix[kind]; + } + }); + } + + return shouldSync; + } + + public shouldJoinChannelsIfNoUsers() { + return this.config.botConfig.joinChannelsIfNoUsers; + } + + public isMembershipListsEnabled() { + return this.config.membershipLists.enabled; + } + + public getUserLocalpart(nick: string) { + // the template is just a literal string with special vars; so find/replace + // the vars and strip the @ + const uid = this.config.matrixClients.userTemplate.replace(/\$SERVER/g, this.domain); + return uid.replace(/\$NICK/g, nick).substring(1); + } + + public claimsUserId(userId: string) { + // the server claims the given user ID if the ID matches the user ID template. + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(userId); + } + + public getNickFromUserId(userId: string) { + // extract the nick from the given user ID + const regex = IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain + }, + { + "$NICK": "(.*?)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(userId); + if (!match) { + return null; + } + return match[1]; + } + + public getUserIdFromNick(nick: string) { + const template = this.config.matrixClients.userTemplate; + return template.replace(/\$NICK/g, nick).replace(/\$SERVER/g, this.domain) + + ":" + this.homeserverDomain; + } + + public getDisplayNameFromNick(nick: string) { + const template = this.config.matrixClients.displayName; + let displayName = template.replace(/\$NICK/g, nick); + displayName = displayName.replace(/\$SERVER/g, this.domain); + return displayName; + } + + public claimsAlias(alias: string) { + // the server claims the given alias if the alias matches the alias template + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "#(.*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + return new RegExp(regex).test(alias); + } + + public getChannelFromAlias(alias: string) { + // extract the channel from the given alias + const regex = IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain + }, + { + "$CHANNEL": "([^:]*)" + }, + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + const match = new RegExp(regex).exec(alias); + if (!match) { + return null; + } + log.info("getChannelFromAlias -> %s -> %s -> %s", alias, regex, match[1]); + return match[1]; + } + + public getAliasFromChannel(channel: string) { + const template = this.config.dynamicChannels.aliasTemplate; + return template.replace(/\$CHANNEL/, channel) + ":" + this.homeserverDomain; + } + + public getNick(userId: string, displayName?: string) { + const illegalChars = BridgedClient.illegalCharactersRegex; + let localpart = userId.substring(1).split(":")[0]; + localpart = localpart.replace(illegalChars, ""); + displayName = displayName ? displayName.replace(illegalChars, "") : undefined; + const display = [displayName, localpart].find((n) => Boolean(n)); + if (!display) { + throw new Error("Could not get nick for user, all characters were invalid"); + } + const template = this.config.ircClients.nickTemplate; + let nick = template.replace(/\$USERID/g, userId); + nick = nick.replace(/\$LOCALPART/g, localpart); + nick = nick.replace(/\$DISPLAY/g, display); + return nick; + } + + public getAliasRegex() { + return IrcServer.templateToRegex( + this.config.dynamicChannels.aliasTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$CHANNEL": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public getUserRegex() { + return IrcServer.templateToRegex( + this.config.matrixClients.userTemplate, + { + "$SERVER": this.domain // find/replace $server + }, + { + "$NICK": ".*" // the nick is unknown, so replace with a wildcard + }, + // Only match the domain of the HS + ":" + IrcServer.escapeRegExp(this.homeserverDomain) + ); + } + + public static get DEFAULT_CONFIG(): IrcServerConfig { + return { + sendConnectionMessages: true, + quitDebounce: { + enabled: false, + quitsPerSecond: 5, + delayMinMs: 3600000, // 1h + delayMaxMs: 7200000, // 2h + }, + botConfig: { + nick: "appservicebot", + joinChannelsIfNoUsers: true, + enabled: true + }, + privateMessages: { + enabled: true, + exclude: [], + federate: true + }, + dynamicChannels: { + enabled: false, + published: true, + createAlias: true, + joinRule: "public", + federate: true, + aliasTemplate: "#irc_$SERVER_$CHANNEL", + whitelist: [], + exclude: [] + }, + mappings: {}, + matrixClients: { + userTemplate: "@$SERVER_$NICK", + displayName: "$NICK (IRC)", + joinAttempts: -1, + }, + ircClients: { + nickTemplate: "M-$DISPLAY", + maxClients: 30, + idleTimeout: 172800, + reconnectIntervalMs: 5000, + concurrentReconnectLimit: 50, + allowNickChanges: false, + ipv6: { + only: false + }, + lineLimit: 3 + }, + membershipLists: { + enabled: false, + floodDelayMs: 10000, // 10s + global: { + ircToMatrix: { + initial: false, + incremental: false + }, + matrixToIrc: { + initial: false, + incremental: false + } + }, + channels: [], + rooms: [] + } + } + } + + private static templateToRegex(template: string, literalVars: {[key: string]: string}, + regexVars: {[key: string]: string}, suffix: string) { + // The 'template' is a literal string with some special variables which need + // to be find/replaced. + let regex = template; + Object.keys(literalVars).forEach(function(varPlaceholder) { + regex = regex.replace( + new RegExp(IrcServer.escapeRegExp(varPlaceholder), 'g'), + literalVars[varPlaceholder] + ); + }); + + // at this point the template is still a literal string, so escape it before + // applying the regex vars. + regex = IrcServer.escapeRegExp(regex); + // apply regex vars + Object.keys(regexVars).forEach(function(varPlaceholder) { + regex = regex.replace( + // double escape, because we bluntly escaped the entire string before + // so our match is now escaped. + new RegExp(IrcServer.escapeRegExp(IrcServer.escapeRegExp(varPlaceholder)), 'g'), + regexVars[varPlaceholder] + ); + }); + + suffix = suffix || ""; + return regex + suffix; + } + + private static escapeRegExp(s: string) { + // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } +} + +export interface IrcServerConfig { + // These are determined to be always defined or possibly undefined + // by the existence of the keys in IrcServer.DEFAULT_CONFIG. + name?: string; + port?: number; + ca?: string; + networkId?: string; + ssl?: boolean; + sslselfsign?: boolean; + sasl?: boolean; + allowExpiredCerts?: boolean; + additionalAddresses?: string[]; + dynamicChannels: { + enabled: boolean; + published: boolean; + createAlias: boolean; + joinRule: "public"|"invite"; + federate: boolean; + aliasTemplate: string; + whitelist: string[]; + exclude: string[]; + roomVersion?: string; + groupId?: string; + }; + quitDebounce: { + enabled: boolean; + quitsPerSecond: number; + delayMinMs: number; + delayMaxMs: number; + }; + mappings: {[channel: string]: string[]}; // chan -> roomId[] + modePowerMap?: {[mode: string]: number}; + sendConnectionMessages: boolean; + botConfig: { + nick: string; + joinChannelsIfNoUsers: boolean; + enabled: boolean; + password?: string; + }; + privateMessages: { + enabled: boolean; + exclude: string[]; + federate: boolean; + }; + matrixClients: { + userTemplate: string; + displayName: string; + joinAttempts: number; + }; + ircClients: { + nickTemplate: string; + maxClients: number; + idleTimeout: number; + reconnectIntervalMs: number; + concurrentReconnectLimit: number; + allowNickChanges: boolean; + ipv6: { + only: boolean; + prefix?: string; + }; + lineLimit: number; + userModes?: string; + }; + membershipLists: { + enabled: boolean; + floodDelayMs: number; + global: { + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }; + channels: { + channel: string; + ircToMatrix: { + initial: boolean; + incremental: boolean; + }; + }[]; + rooms: { + room: string; + matrixToIrc: { + initial: boolean; + incremental: boolean; + }; + }[]; + }; +} diff --git a/src/main.js b/src/main.js index 1bec2fa5a..0a5669566 100644 --- a/src/main.js +++ b/src/main.js @@ -7,7 +7,7 @@ const RoomBridgeStore = require("matrix-appservice-bridge").RoomBridgeStore; const UserBridgeStore = require("matrix-appservice-bridge").UserBridgeStore; const IrcBridge = require("./bridge/IrcBridge.js"); -const IrcServer = require("./irc/IrcServer.js"); +const { IrcServer } = require("./irc/IrcServer.js"); const stats = require("./config/stats"); const ident = require("./irc/ident"); const logging = require("./logging"); diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index e90881b37..e51ac7e18 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -17,6 +17,7 @@ limitations under the License. //@ts-ignore import { RemoteRoom } from "matrix-appservice-bridge"; import { toIrcLowerCase } from "../irc/formatting"; +import { IrcServer } from "../irc/IrcServer"; export class IrcRoom extends RemoteRoom { /** @@ -26,7 +27,7 @@ export class IrcRoom extends RemoteRoom { * @param {String} channel : The channel this room represents. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(public readonly server: any, public readonly channel: string) { + constructor(public readonly server: IrcServer, public readonly channel: string) { // Because `super` must be called first, we convert the case several times. super(IrcRoom.createId(server, toIrcLowerCase(channel)), { domain: server.domain, @@ -40,7 +41,7 @@ export class IrcRoom extends RemoteRoom { } getDomain() { - return super.get("domain"); + return super.get("domain") as string; } getServer() { @@ -48,17 +49,17 @@ export class IrcRoom extends RemoteRoom { } getChannel() { - return super.get("channel"); + return super.get("channel") as string; } getType() { - return super.get("type"); + return super.get("type") as string; } // No types for IrcServer yet // eslint-disable-next-line @typescript-eslint/no-explicit-any public static fromRemoteRoom(server: any, remoteRoom: RemoteRoom) { - return new IrcRoom(server, remoteRoom.get("channel")); + return new IrcRoom(server, remoteRoom.get("channel") as string); } // An IRC room is uniquely identified by a combination of the channel name and the From 723231464b4fe189fe19c63c3ced0b925952d7da Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 11:17:06 +0100 Subject: [PATCH 041/350] Add option to exclude users from being bridged --- config.sample.yaml | 4 ++++ src/bridge/IrcBridge.js | 6 ++++++ src/bridge/MatrixHandler.js | 5 ++++- src/irc/ClientPool.js | 2 +- src/irc/IrcServer.ts | 22 +++++++++++++++++++++- 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 7dbbef3dc..949202703 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -203,6 +203,10 @@ ircService: # circumstances. # exclude: ["#foo", "#bar"] + # excludedUsers: + # - regex: "@.*:evilcorp.com" + # kickReason: "We don't like Evilcorp" + # Configuration for controlling how Matrix and IRC membership lists are # synced. membershipLists: diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index edcd08e6b..336b900e4 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -297,6 +297,12 @@ IrcBridge.prototype.createBridgedClient = function(ircClientConfig, matrixUser, ); } + const excluded = this.server.isExcludedUser(matrixUser.getId()); + + if (excluded) { + throw Error("Cannot create bridged client - user is excluded from bridging"); + } + return new BridgedClient( server, ircClientConfig, matrixUser, isBot, this._ircEventBroker, this._identGenerator, this._ipv6Generator diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index a9ad259b5..22eeb449e 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -958,9 +958,12 @@ MatrixHandler.prototype._onJoin = Promise.coroutine(function*(req, event, user) while (kickIntent) { try { + // If they are known blacklisted, get a specific reason string. + const excluded = server.isExcludedUser(user.getId()); yield kickIntent.kick( event.room_id, user.getId(), - `IRC connection failure.` + excluded && excluded.kickReason ? excluded.kickReason + : `IRC connection failure.`, ); self._incrementMetric(room.server.domain, "connection_failure_kicks"); break; diff --git a/src/irc/ClientPool.js b/src/irc/ClientPool.js index 4858dc816..627ec476b 100644 --- a/src/irc/ClientPool.js +++ b/src/irc/ClientPool.js @@ -117,7 +117,7 @@ ClientPool.prototype.getBot = function(server) { }; ClientPool.prototype.createIrcClient = function(ircClientConfig, matrixUser, isBot) { - var bridgedClient = this._ircBridge.createBridgedClient( + const bridgedClient = this._ircBridge.createBridgedClient( ircClientConfig, matrixUser, isBot ); var server = bridgedClient.server; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 40cf1ef92..f2bbefb21 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -14,6 +14,7 @@ type MembershipSyncKind = "incremental"|"initial"; export class IrcServer { private addresses: string[]; private groupIdValid: boolean; + private excludedUsers: { regex: RegExp; kickReason?: string; }[]; /** * Construct a new IRC Server. * @constructor @@ -29,6 +30,12 @@ export class IrcServer { private homeserverDomain: string, private expiryTimeSeconds: number = 0) { this.addresses = config.additionalAddresses || []; this.addresses.push(domain); + this.excludedUsers = config.excludedUsers.map((excluded) => { + return { + ...excluded, + regex: new RegExp(excluded.regex) + } + }) if (this.config.dynamicChannels.groupId !== undefined && this.config.dynamicChannels.groupId.trim() !== "") { @@ -237,7 +244,7 @@ export class IrcServer { public createBotIrcClientConfig(username: string) { return IrcClientConfig.newConfig( null, this.domain, this.config.botConfig.nick, username, - this.config.botConfig.password + this.config.botConfig.password! ); } @@ -261,6 +268,12 @@ export class IrcServer { return this.config.dynamicChannels.exclude.indexOf(channel) !== -1; } + public isExcludedUser(userId: string) { + return this.excludedUsers.find((exclusion) => { + return exclusion.regex.exec(userId) !== null; + }); + } + public hasInviteRooms() { return ( this.config.dynamicChannels.enabled && this.getJoinRule() === "invite" @@ -507,6 +520,7 @@ export class IrcServer { exclude: [] }, mappings: {}, + excludedUsers: [], matrixClients: { userTemplate: "@$SERVER_$NICK", displayName: "$NICK (IRC)", @@ -641,6 +655,12 @@ export interface IrcServerConfig { lineLimit: number; userModes?: string; }; + excludedUsers: Array< + { + regex: string; + kickReason?: string; + } + >; membershipLists: { enabled: boolean; floodDelayMs: number; From 3a5cd453003a3054ca256c4719d34a8464c6259b Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:16:14 +0100 Subject: [PATCH 042/350] Add test to ensure users are excluded --- spec/integ/irc-connections.spec.js | 45 +++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/spec/integ/irc-connections.spec.js b/spec/integ/irc-connections.spec.js index 63dea881f..9c3f676cc 100644 --- a/spec/integ/irc-connections.spec.js +++ b/spec/integ/irc-connections.spec.js @@ -6,12 +6,19 @@ const Promise = require("bluebird"); const envBundle = require("../util/env-bundle"); describe("IRC connections", function() { + let testUser = { id: "@alice:hs", nick: "M-alice" }; const {env, config, roomMapping, test} = envBundle(); + // Ensure the right users are excluded. + Object.values(config.ircService.servers)[0].excludedUsers = [ + { + regex: "@excluded:hs", + } + ]; beforeEach(test.coroutine(function*() { yield test.beforeEach(env); @@ -30,7 +37,7 @@ describe("IRC connections", function() { ); // do the init - yield test.initEnv(env); + yield test.initEnv(env, config); })); afterEach(test.coroutine(function*() { @@ -520,4 +527,40 @@ describe("IRC connections", function() { done(); }); }); + + it("should not bridge matrix users who are excluded", async function() { + const excludedUserId = "@excluded:hs"; + const nick = "M-excluded"; + + env.ircMock._whenClient(roomMapping.server, nick, "connect", + function() { + throw Error("Client should not be saying anything") + }); + + const botSdk = env.clientMock._client(config._botUserId); + botSdk.kick.and.callFake(async (roomId, userId, reason) => { + if (roomId === roomMapping.roomId && userId === excludedUserId) { + throw Error("Should not kick"); + } + }); + + try { + await env.mockAppService._trigger("type:m.room.message", { + content: { + body: "Text that should never be sent", + msgtype: "m.text" + }, + user_id: excludedUserId, + room_id: roomMapping.roomId, + type: "m.room.message" + }); + } + catch (ex) { + expect(ex.message).toBe( + "Cannot create bridged client - user is excluded from bridging" + ); + return; + } + throw Error("Should have thrown"); + }); }); From 00648a146b3534b6b3b06402adab4a65ac8678c5 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:16:43 +0100 Subject: [PATCH 043/350] Tweaks to make cherrypicked IrcServer work --- src/bridge/IrcBridge.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 336b900e4..c7aca980d 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -18,8 +18,7 @@ const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); var BridgeRequest = require("../models/BridgeRequest"); var stats = require("../config/stats"); -const { NeDBDataStore } = require("../datastore/NedbDataStore"); -const { PgDataStore } = require("../datastore/postgres/PgDataStore"); +const { DataStore } = require("../DataStore"); var log = require("../logging").get("IrcBridge"); const { Bridge, @@ -297,10 +296,11 @@ IrcBridge.prototype.createBridgedClient = function(ircClientConfig, matrixUser, ); } - const excluded = this.server.isExcludedUser(matrixUser.getId()); - - if (excluded) { - throw Error("Cannot create bridged client - user is excluded from bridging"); + if (matrixUser) { // Don't bother with the bot user + const excluded = server.isExcludedUser(matrixUser.userId); + if (excluded) { + throw Error("Cannot create bridged client - user is excluded from bridging"); + } } return new BridgedClient( @@ -325,12 +325,12 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; - this._dataStore = new NeDBDataStore( - this._bridge.getUserStore(), - this._bridge.getRoomStore(), - pkeyPath, - this.config.homeserver.domain, - ); + this._dataStore = new DataStore( + this._bridge.getUserStore(), + this._bridge.getRoomStore(), + pkeyPath, + this.config.homeserver.domain, + ); yield this._dataStore.removeConfigMappings(); this._identGenerator = new IdentGenerator(this._dataStore); @@ -1004,9 +1004,8 @@ IrcBridge.prototype.getBridgedClient = Promise.coroutine(function*(server, userI "Creating virtual irc user with nick %s for %s (display name %s)", ircClientConfig.getDesiredNick(), userId, displayName ); - bridgedClient = this._clientPool.createIrcClient(ircClientConfig, mxUser, false); - try { + bridgedClient = this._clientPool.createIrcClient(ircClientConfig, mxUser, false); yield bridgedClient.connect(); if (!storedConfig) { yield this.getStore().storeIrcClientConfig(ircClientConfig); From 3f7ff21c7345e49e0f7e6307064bfb345994adc1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:18:05 +0100 Subject: [PATCH 044/350] Move schema out of source tree --- app.js | 2 +- config.schema.yml | 323 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 config.schema.yml diff --git a/app.js b/app.js index da9412cd1..cc8d3c4f5 100644 --- a/app.js +++ b/app.js @@ -12,7 +12,7 @@ new Cli({ enableLocalpart: true, bridgeConfig: { affectsRegistration: true, - schema: path.join(__dirname, "lib/config/schema.yml"), + schema: path.join(__dirname, "config.schema.yml"), defaults: { homeserver: { dropMatrixMessagesAfterSecs: 0, diff --git a/config.schema.yml b/config.schema.yml new file mode 100644 index 000000000..8a102cd78 --- /dev/null +++ b/config.schema.yml @@ -0,0 +1,323 @@ +"$schema": "http://json-schema.org/draft-04/schema#" +type: "object" +properties: + advanced: + type: "object" + properties: + maxHttpSockets: + type: "integer" + homeserver: + type: "object" + properties: + url: + type: "string" + media_url: + type: "string" + domain: + type: "string" + dropMatrixMessagesAfterSecs: + type: "integer" + enablePresence: + type: "boolean" + required: ["url", "domain"] + ircService: + type: "object" + properties: + databaseUri: + type: "string" + metrics: + type: "object" + properties: + enabled: + type: "boolean" + remoteUserAgeBuckets: + type: "array" + items: + type: "string" + pattern: "^[0-9]+(h|d|w)$" + statsd: + type: "object" + properties: + hostname: + type: "string" + port: + type: "integer" + jobName: + type: "string" + required: ["hostname", "port"] + ident: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + address: + type: "string" + required: ["enabled"] + debugApi: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + required: ["enabled", "port"] + logging: + type: "object" + properties: + level: + type: "string" + enum: ["error","warn","info","debug"] + logfile: + type: "string" + errfile: + type: "string" + toConsole: + type: "boolean" + maxFileSizeBytes: + type: "integer" + maxFiles: + type: "integer" + provisioning: + type: "object" + properties: + enabled: + type: "boolean" + requestTimeoutSeconds: + type: "number" + ruleFile: + type: "string" + enableReload: + type: "boolean" + passwordEncryptionKeyPath: + type: "string" + matrixHandler: + type: "object" + properties: + eventCacheSize: + type: "integer" + ircHandler: + type: "object" + properties: + leaveConcurrency: + type: "integer" + mapIrcMentionsToMatrix: + type: "string" + enum: ["on", "off", "force-off"] + servers: + type: "object" + # all properties must follow the following + additionalProperties: + type: "object" + properties: + port: + type: "integer" + additionalAddresses: + type: "array" + items: + type: "string" + ssl: + type: "boolean" + sslselfsign: + type: "boolean" + sasl: + type: "boolean" + allowExpiredCerts: + type: "boolean" + password: + type: "string" + sendConnectionMessages: + type: "boolean" + name: + type: "string" + description: + type: "string" + networkId: + type: "string" + pattern: "^[a-zA-Z0-9]+$" + icon: + type: "string" + quitDebounce: + type: "object" + properties: + enabled: + type: "boolean" + quitsPerSecond: + type: "number" + delayMinMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + delayMaxMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + modePowerMap: + type: "object" + patternProperties: + # Single character modes mapped to positive power levels + "^[a-zA-Z]$": + type: number + minimum: 0 + botConfig: + type: "object" + properties: + enabled: + type: "boolean" + nick: + type: "string" + password: + type: "string" + joinChannelsIfNoUsers: + type: "boolean" + privateMessages: + type: "object" + properties: + enabled: + type: "boolean" + exclude: + type: "array" + items: + type: "string" + federate: + type: "boolean" + membershipLists: + type: "object" + properties: + enabled: + type: "boolean" + floodDelayMs: + type: "integer" + global: + type: "object" + properties: + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + rooms: + type: "array" + items: + type: "object" + properties: + room: + type: "string" + pattern: "^!+.*$" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + channels: + type: "array" + items: + type: "object" + properties: + channel: + type: "string" + pattern: "^#+.*$" + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + dynamicChannels: + type: "object" + properties: + enabled: + type: "boolean" + published: + type: "boolean" + createAlias: + type: "boolean" + groupId: + type: "string" + joinRule: + type: "string" + enum: ["invite", "public"] + federate: + type: "boolean" + roomVersion: + type: "string" + aliasTemplate: + type: "string" + pattern: "^#.*\\$CHANNEL" + whitelist: + type: "array" + items: + type: "string" + pattern: "^@.*" + exclude: + type: "array" + items: + type: "string" + mappings: + type: "object" + patternProperties: + # must start with a # + "^#+.*$": + type: "array" + items: + type: "string" + minItems: 1 + uniqueItems: true + additionalProperties: false + matrixClients: + type: "object" + properties: + userTemplate: + type: "string" + pattern: "^@.*\\$NICK" + displayName: + type: "string" + pattern: "\\$NICK" + joinAttempts: + type: "integer" + minimum: -1 + ircClients: + type: "object" + properties: + nickTemplate: + type: "string" + pattern: "\\$USERID|\\$LOCALPART|\\$DISPLAY" + maxClients: + type: "integer" + idleTimeout: + type: "integer" + minimum: 0 + reconnectIntervalMs: + type: "integer" + minimum: 0 + allowNickChanges: + type: "boolean" + ipv6: + type: "object" + properties: + prefix: + type: "string" + pattern: "[ABCDEFabcdef0123456789:]+" + only: + type: "boolean" + lineLimit: + type: "integer" + userModes: + type: "string" + required: ["databaseUri", "servers"] From b9f850fb5a282bf9bb8758141fe04f5ba979bf08 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:24:02 +0100 Subject: [PATCH 045/350] Move schema into / --- app.js | 2 +- schema.yml | 323 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 schema.yml diff --git a/app.js b/app.js index da9412cd1..cc8d3c4f5 100644 --- a/app.js +++ b/app.js @@ -12,7 +12,7 @@ new Cli({ enableLocalpart: true, bridgeConfig: { affectsRegistration: true, - schema: path.join(__dirname, "lib/config/schema.yml"), + schema: path.join(__dirname, "config.schema.yml"), defaults: { homeserver: { dropMatrixMessagesAfterSecs: 0, diff --git a/schema.yml b/schema.yml new file mode 100644 index 000000000..8a102cd78 --- /dev/null +++ b/schema.yml @@ -0,0 +1,323 @@ +"$schema": "http://json-schema.org/draft-04/schema#" +type: "object" +properties: + advanced: + type: "object" + properties: + maxHttpSockets: + type: "integer" + homeserver: + type: "object" + properties: + url: + type: "string" + media_url: + type: "string" + domain: + type: "string" + dropMatrixMessagesAfterSecs: + type: "integer" + enablePresence: + type: "boolean" + required: ["url", "domain"] + ircService: + type: "object" + properties: + databaseUri: + type: "string" + metrics: + type: "object" + properties: + enabled: + type: "boolean" + remoteUserAgeBuckets: + type: "array" + items: + type: "string" + pattern: "^[0-9]+(h|d|w)$" + statsd: + type: "object" + properties: + hostname: + type: "string" + port: + type: "integer" + jobName: + type: "string" + required: ["hostname", "port"] + ident: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + address: + type: "string" + required: ["enabled"] + debugApi: + type: "object" + properties: + enabled: + type: "boolean" + port: + type: "integer" + required: ["enabled", "port"] + logging: + type: "object" + properties: + level: + type: "string" + enum: ["error","warn","info","debug"] + logfile: + type: "string" + errfile: + type: "string" + toConsole: + type: "boolean" + maxFileSizeBytes: + type: "integer" + maxFiles: + type: "integer" + provisioning: + type: "object" + properties: + enabled: + type: "boolean" + requestTimeoutSeconds: + type: "number" + ruleFile: + type: "string" + enableReload: + type: "boolean" + passwordEncryptionKeyPath: + type: "string" + matrixHandler: + type: "object" + properties: + eventCacheSize: + type: "integer" + ircHandler: + type: "object" + properties: + leaveConcurrency: + type: "integer" + mapIrcMentionsToMatrix: + type: "string" + enum: ["on", "off", "force-off"] + servers: + type: "object" + # all properties must follow the following + additionalProperties: + type: "object" + properties: + port: + type: "integer" + additionalAddresses: + type: "array" + items: + type: "string" + ssl: + type: "boolean" + sslselfsign: + type: "boolean" + sasl: + type: "boolean" + allowExpiredCerts: + type: "boolean" + password: + type: "string" + sendConnectionMessages: + type: "boolean" + name: + type: "string" + description: + type: "string" + networkId: + type: "string" + pattern: "^[a-zA-Z0-9]+$" + icon: + type: "string" + quitDebounce: + type: "object" + properties: + enabled: + type: "boolean" + quitsPerSecond: + type: "number" + delayMinMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + delayMaxMs: + type: "integer" + minimum: 0 + exclusiveMinimum: true + modePowerMap: + type: "object" + patternProperties: + # Single character modes mapped to positive power levels + "^[a-zA-Z]$": + type: number + minimum: 0 + botConfig: + type: "object" + properties: + enabled: + type: "boolean" + nick: + type: "string" + password: + type: "string" + joinChannelsIfNoUsers: + type: "boolean" + privateMessages: + type: "object" + properties: + enabled: + type: "boolean" + exclude: + type: "array" + items: + type: "string" + federate: + type: "boolean" + membershipLists: + type: "object" + properties: + enabled: + type: "boolean" + floodDelayMs: + type: "integer" + global: + type: "object" + properties: + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + rooms: + type: "array" + items: + type: "object" + properties: + room: + type: "string" + pattern: "^!+.*$" + matrixToIrc: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + channels: + type: "array" + items: + type: "object" + properties: + channel: + type: "string" + pattern: "^#+.*$" + ircToMatrix: + type: "object" + properties: + initial: + type: "boolean" + incremental: + type: "boolean" + additionalProperties: false + dynamicChannels: + type: "object" + properties: + enabled: + type: "boolean" + published: + type: "boolean" + createAlias: + type: "boolean" + groupId: + type: "string" + joinRule: + type: "string" + enum: ["invite", "public"] + federate: + type: "boolean" + roomVersion: + type: "string" + aliasTemplate: + type: "string" + pattern: "^#.*\\$CHANNEL" + whitelist: + type: "array" + items: + type: "string" + pattern: "^@.*" + exclude: + type: "array" + items: + type: "string" + mappings: + type: "object" + patternProperties: + # must start with a # + "^#+.*$": + type: "array" + items: + type: "string" + minItems: 1 + uniqueItems: true + additionalProperties: false + matrixClients: + type: "object" + properties: + userTemplate: + type: "string" + pattern: "^@.*\\$NICK" + displayName: + type: "string" + pattern: "\\$NICK" + joinAttempts: + type: "integer" + minimum: -1 + ircClients: + type: "object" + properties: + nickTemplate: + type: "string" + pattern: "\\$USERID|\\$LOCALPART|\\$DISPLAY" + maxClients: + type: "integer" + idleTimeout: + type: "integer" + minimum: 0 + reconnectIntervalMs: + type: "integer" + minimum: 0 + allowNickChanges: + type: "boolean" + ipv6: + type: "object" + properties: + prefix: + type: "string" + pattern: "[ABCDEFabcdef0123456789:]+" + only: + type: "boolean" + lineLimit: + type: "integer" + userModes: + type: "string" + required: ["databaseUri", "servers"] From 44e63bd2f399e3d2de28771b4a42350ca56fada5 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:35:34 +0100 Subject: [PATCH 046/350] Update schema --- config.schema.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config.schema.yml b/config.schema.yml index 8a102cd78..ec41921c4 100644 --- a/config.schema.yml +++ b/config.schema.yml @@ -320,4 +320,12 @@ properties: type: "integer" userModes: type: "string" + excludeUsers: + type: "array" + properties: + regex: + type: "string" + kickReason: + type: "string" + required: ["regex"] required: ["databaseUri", "servers"] From 5b033e467832abedde7631f7ef98f3258531fc19 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 13:36:33 +0100 Subject: [PATCH 047/350] Fix merge --- src/irc/IrcServer.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 2966d365d..f2bbefb21 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -14,10 +14,7 @@ type MembershipSyncKind = "incremental"|"initial"; export class IrcServer { private addresses: string[]; private groupIdValid: boolean; -<<<<<<< HEAD private excludedUsers: { regex: RegExp; kickReason?: string; }[]; -======= ->>>>>>> hs/move-schema /** * Construct a new IRC Server. * @constructor @@ -33,15 +30,12 @@ export class IrcServer { private homeserverDomain: string, private expiryTimeSeconds: number = 0) { this.addresses = config.additionalAddresses || []; this.addresses.push(domain); -<<<<<<< HEAD this.excludedUsers = config.excludedUsers.map((excluded) => { return { ...excluded, regex: new RegExp(excluded.regex) } }) -======= ->>>>>>> hs/move-schema if (this.config.dynamicChannels.groupId !== undefined && this.config.dynamicChannels.groupId.trim() !== "") { @@ -526,10 +520,7 @@ export class IrcServer { exclude: [] }, mappings: {}, -<<<<<<< HEAD excludedUsers: [], -======= ->>>>>>> hs/move-schema matrixClients: { userTemplate: "@$SERVER_$NICK", displayName: "$NICK (IRC)", @@ -664,15 +655,12 @@ export interface IrcServerConfig { lineLimit: number; userModes?: string; }; -<<<<<<< HEAD excludedUsers: Array< { regex: string; kickReason?: string; } >; -======= ->>>>>>> hs/move-schema membershipLists: { enabled: boolean; floodDelayMs: number; From a1e89fb7ca943cd133d7d9a0cd9ecd77a575734d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 14:11:47 +0100 Subject: [PATCH 048/350] We don't need to specify all the values here --- src/datastore/NedbDataStore.ts | 4 ---- types/matrix-appservice-bridge/index.d.ts | 9 ++++++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index ab939db6f..b82f0c3af 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -467,11 +467,7 @@ export class NeDBDataStore implements DataStore { room.set("admin_id", userId); await this.roomStore.upsertEntry({ id: NeDBDataStore.createAdminId(userId), - matrix_id: room.getId(), matrix: room, - remote: null, - remote_id: "", - data: {}, }); } diff --git a/types/matrix-appservice-bridge/index.d.ts b/types/matrix-appservice-bridge/index.d.ts index a4401628d..1ec9e451c 100644 --- a/types/matrix-appservice-bridge/index.d.ts +++ b/types/matrix-appservice-bridge/index.d.ts @@ -42,6 +42,13 @@ declare module 'matrix-appservice-bridge' { data: null|any // Information about this mapping, which may be an empty. } + export interface UpsertableEntry { + id: string // The unique ID for this entry. + matrix?: null|MatrixRoom // The matrix room, if applicable. + remote?: null|RemoteRoom // The remote room, if applicable. + data?: null|any // Information about this mapping, which may be an empty. + } + export class MatrixRoom { protected roomId: string @@ -110,7 +117,7 @@ declare module 'matrix-appservice-bridge' { removeEntriesByRemoteRoomData (data: object): Promise removeEntriesByRemoteRoomId (remoteId: string): Promise setMatrixRoom (matrixRoom: MatrixRoom): Promise - upsertEntry (entry: Entry): Promise + upsertEntry (entry: UpsertableEntry): Promise linkRooms ( matrixRoom: MatrixRoom, remoteRoom: RemoteRoom, From c5a974a67d5473d8f86c9a953bf72ab4c71c5866 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 14:43:05 +0100 Subject: [PATCH 049/350] Use towncrier --- .buildkite/pipeline.yml | 11 +++++++++++ changelog.d/821.misc | 1 + pyproject.toml | 30 ++++++++++++++++++++++++++++++ scripts/towncrier.sh | 3 +++ 4 files changed, 45 insertions(+) create mode 100644 changelog.d/821.misc create mode 100644 pyproject.toml create mode 100644 scripts/towncrier.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1923243cc..c190abb6b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -33,3 +33,14 @@ steps: plugins: - docker#v3.0.1: image: "node:12" + + # Borrowed from https://github.com/matrix-org/synapse/blob/master/.buildkite/pipeline.yml + - label: ":newspaper: Newsfile" + branches: "!master !develop !release-*" + command: + - "python3 -m pip install towncrier" + - "python3 -m towncrier.check --compare-with=origin/develop" + plugins: + - docker#v3.0.1: + image: "python:3.6" + propagate-environment: true \ No newline at end of file diff --git a/changelog.d/821.misc b/changelog.d/821.misc new file mode 100644 index 000000000..a07ab4328 --- /dev/null +++ b/changelog.d/821.misc @@ -0,0 +1 @@ +Use [Towncrier](https://pypi.org/project/towncrier/) for changelog management \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7f2761a6e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + # The name of your Python package + filename = "CHANGELOG.md" + directory = "changelog.d" + issue_format = "[\\#{issue}](https://github.com/matrix-org/matrix-appservice-irc/issues/{issue})" + + [[tool.towncrier.type]] + directory = "feature" + name = "Features" + showcontent = true + + [[tool.towncrier.type]] + directory = "bugfix" + name = "Bugfixes" + showcontent = true + + [[tool.towncrier.type]] + directory = "doc" + name = "Improved Documentation" + showcontent = true + + [[tool.towncrier.type]] + directory = "removal" + name = "Deprecations and Removals" + showcontent = true + + [[tool.towncrier.type]] + directory = "misc" + name = "Internal Changes" + showcontent = true diff --git a/scripts/towncrier.sh b/scripts/towncrier.sh new file mode 100644 index 000000000..c5b3d8d83 --- /dev/null +++ b/scripts/towncrier.sh @@ -0,0 +1,3 @@ +#!/bin/bash +VERSION=`python3 -c "import json; f = open('./package.json', 'r'); v = json.loads(f.read())['version']; f.close(); print(v)"` +towncrier --version $VERSION $1 \ No newline at end of file From a331c016fbefe8b86cc569991d5b1148d0654fe6 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 14:44:50 +0100 Subject: [PATCH 050/350] Include changelog helper text --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6e2a2c4a..239309a6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ This project follows "git flow" semantics. In practice, this means: `npm run check`. - Create a pull request. If this PR fixes an issue, link to it by referring to its number. - PRs from community members must be signed off as per Synapse's [Attribution section](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#attribution) + - Create a changelog entry in `changelog.d`. A changelog filename should be `${GithubIssueNumber}.{bugfix|misc|feature|doc|removal}` + The change should include information that is useful to the user rather than the developer. ## Coding notes The IRC bridge is compatible on Node.js v10+. Buildkite is used to ensure that tests will run on From a313d57efdc62d3d50171f9543805b19fd33cbde Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 15:47:37 +0100 Subject: [PATCH 051/350] Tidyup --- src/datastore/DataStore.ts | 18 +++++++++++++++++- src/datastore/NedbDataStore.ts | 2 +- src/datastore/StringCrypto.ts | 16 ++++++++++++++++ src/irc/IrcServer.ts | 17 ++++++++++++++++- src/models/IrcClientConfig.ts | 2 -- src/models/IrcRoom.ts | 1 - 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index b051957f1..ad3c07584 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -1,5 +1,21 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import { MatrixRoom, MatrixUser, Entry} from "matrix-appservice-bridge"; -import {default as Bluebird} from "bluebird"; +import Bluebird from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index b82f0c3af..489477933 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {default as Bluebird} from "bluebird"; +import Bluebird from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig, IrcClientConfigSeralized } from "../models/IrcClientConfig" import * as logging from "../logging"; diff --git a/src/datastore/StringCrypto.ts b/src/datastore/StringCrypto.ts index b294a786c..a6c0106b2 100644 --- a/src/datastore/StringCrypto.ts +++ b/src/datastore/StringCrypto.ts @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import * as crypto from "crypto"; import * as fs from "fs"; import * as logging from "../logging"; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index ac7a6d16f..37190cbba 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -1,3 +1,18 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ import * as logging from "../logging"; import * as BridgedClient from "./BridgedClient"; @@ -14,7 +29,7 @@ type MembershipSyncKind = "incremental"|"initial"; export class IrcServer { private addresses: string[]; private groupIdValid: boolean; - private excludedUsers: { regex: RegExp; kickReason?: string; }[]; + private excludedUsers: { regex: RegExp; kickReason?: string }[]; /** * Construct a new IRC Server. * @constructor diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index 46d2b1c59..89b30022c 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Ignore definition errors for now. -//@ts-ignore import { MatrixUser } from "matrix-appservice-bridge"; export interface IrcClientConfigSeralized { diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index e51ac7e18..7db323000 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -//@ts-ignore import { RemoteRoom } from "matrix-appservice-bridge"; import { toIrcLowerCase } from "../irc/formatting"; import { IrcServer } from "../irc/IrcServer"; From 5ed37253f0cbc2f2c9f3102fa1f83ffb05c0c8db Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:10:49 +0100 Subject: [PATCH 052/350] Remove unused schema.yml --- schema.yml | 323 ----------------------------------------------------- 1 file changed, 323 deletions(-) delete mode 100644 schema.yml diff --git a/schema.yml b/schema.yml deleted file mode 100644 index 8a102cd78..000000000 --- a/schema.yml +++ /dev/null @@ -1,323 +0,0 @@ -"$schema": "http://json-schema.org/draft-04/schema#" -type: "object" -properties: - advanced: - type: "object" - properties: - maxHttpSockets: - type: "integer" - homeserver: - type: "object" - properties: - url: - type: "string" - media_url: - type: "string" - domain: - type: "string" - dropMatrixMessagesAfterSecs: - type: "integer" - enablePresence: - type: "boolean" - required: ["url", "domain"] - ircService: - type: "object" - properties: - databaseUri: - type: "string" - metrics: - type: "object" - properties: - enabled: - type: "boolean" - remoteUserAgeBuckets: - type: "array" - items: - type: "string" - pattern: "^[0-9]+(h|d|w)$" - statsd: - type: "object" - properties: - hostname: - type: "string" - port: - type: "integer" - jobName: - type: "string" - required: ["hostname", "port"] - ident: - type: "object" - properties: - enabled: - type: "boolean" - port: - type: "integer" - address: - type: "string" - required: ["enabled"] - debugApi: - type: "object" - properties: - enabled: - type: "boolean" - port: - type: "integer" - required: ["enabled", "port"] - logging: - type: "object" - properties: - level: - type: "string" - enum: ["error","warn","info","debug"] - logfile: - type: "string" - errfile: - type: "string" - toConsole: - type: "boolean" - maxFileSizeBytes: - type: "integer" - maxFiles: - type: "integer" - provisioning: - type: "object" - properties: - enabled: - type: "boolean" - requestTimeoutSeconds: - type: "number" - ruleFile: - type: "string" - enableReload: - type: "boolean" - passwordEncryptionKeyPath: - type: "string" - matrixHandler: - type: "object" - properties: - eventCacheSize: - type: "integer" - ircHandler: - type: "object" - properties: - leaveConcurrency: - type: "integer" - mapIrcMentionsToMatrix: - type: "string" - enum: ["on", "off", "force-off"] - servers: - type: "object" - # all properties must follow the following - additionalProperties: - type: "object" - properties: - port: - type: "integer" - additionalAddresses: - type: "array" - items: - type: "string" - ssl: - type: "boolean" - sslselfsign: - type: "boolean" - sasl: - type: "boolean" - allowExpiredCerts: - type: "boolean" - password: - type: "string" - sendConnectionMessages: - type: "boolean" - name: - type: "string" - description: - type: "string" - networkId: - type: "string" - pattern: "^[a-zA-Z0-9]+$" - icon: - type: "string" - quitDebounce: - type: "object" - properties: - enabled: - type: "boolean" - quitsPerSecond: - type: "number" - delayMinMs: - type: "integer" - minimum: 0 - exclusiveMinimum: true - delayMaxMs: - type: "integer" - minimum: 0 - exclusiveMinimum: true - modePowerMap: - type: "object" - patternProperties: - # Single character modes mapped to positive power levels - "^[a-zA-Z]$": - type: number - minimum: 0 - botConfig: - type: "object" - properties: - enabled: - type: "boolean" - nick: - type: "string" - password: - type: "string" - joinChannelsIfNoUsers: - type: "boolean" - privateMessages: - type: "object" - properties: - enabled: - type: "boolean" - exclude: - type: "array" - items: - type: "string" - federate: - type: "boolean" - membershipLists: - type: "object" - properties: - enabled: - type: "boolean" - floodDelayMs: - type: "integer" - global: - type: "object" - properties: - ircToMatrix: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - matrixToIrc: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - rooms: - type: "array" - items: - type: "object" - properties: - room: - type: "string" - pattern: "^!+.*$" - matrixToIrc: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - channels: - type: "array" - items: - type: "object" - properties: - channel: - type: "string" - pattern: "^#+.*$" - ircToMatrix: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - dynamicChannels: - type: "object" - properties: - enabled: - type: "boolean" - published: - type: "boolean" - createAlias: - type: "boolean" - groupId: - type: "string" - joinRule: - type: "string" - enum: ["invite", "public"] - federate: - type: "boolean" - roomVersion: - type: "string" - aliasTemplate: - type: "string" - pattern: "^#.*\\$CHANNEL" - whitelist: - type: "array" - items: - type: "string" - pattern: "^@.*" - exclude: - type: "array" - items: - type: "string" - mappings: - type: "object" - patternProperties: - # must start with a # - "^#+.*$": - type: "array" - items: - type: "string" - minItems: 1 - uniqueItems: true - additionalProperties: false - matrixClients: - type: "object" - properties: - userTemplate: - type: "string" - pattern: "^@.*\\$NICK" - displayName: - type: "string" - pattern: "\\$NICK" - joinAttempts: - type: "integer" - minimum: -1 - ircClients: - type: "object" - properties: - nickTemplate: - type: "string" - pattern: "\\$USERID|\\$LOCALPART|\\$DISPLAY" - maxClients: - type: "integer" - idleTimeout: - type: "integer" - minimum: 0 - reconnectIntervalMs: - type: "integer" - minimum: 0 - allowNickChanges: - type: "boolean" - ipv6: - type: "object" - properties: - prefix: - type: "string" - pattern: "[ABCDEFabcdef0123456789:]+" - only: - type: "boolean" - lineLimit: - type: "integer" - userModes: - type: "string" - required: ["databaseUri", "servers"] From 237d3c96b3d0adb2745417ee99265e08536fa112 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:11:33 +0100 Subject: [PATCH 053/350] Newsfile --- changelog.d/815.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/815.feature diff --git a/changelog.d/815.feature b/changelog.d/815.feature new file mode 100644 index 000000000..47133a9b8 --- /dev/null +++ b/changelog.d/815.feature @@ -0,0 +1 @@ +Add support for PostgreSQL \ No newline at end of file From 83d2446dca4b319c976bb5a83dc7dcbd6fd70016 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:15:39 +0100 Subject: [PATCH 054/350] Package version --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index b6bdf8fc7..95d552800 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "matrix-appservice-irc", - "version": "0.12.0", + "version": "0.13.0", "lockfileVersion": 1, "requires": true, "dependencies": { From b5b2ce0c4d45b3a37e4c24912d0966635d387336 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:17:56 +0100 Subject: [PATCH 055/350] Newsfile --- changelog.d/816.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/816.feature diff --git a/changelog.d/816.feature b/changelog.d/816.feature new file mode 100644 index 000000000..d439b9500 --- /dev/null +++ b/changelog.d/816.feature @@ -0,0 +1 @@ +Add migration script to allow migration from NeDB to PostgreSQL \ No newline at end of file From 389ddeaf7c5646fd3c2e5433b866127b917b8153 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:38:00 +0100 Subject: [PATCH 056/350] Convert models/BridgeRequest to Typescript --- src/DebugApi.js | 2 +- src/bridge/IrcBridge.js | 2 +- src/bridge/IrcHandler.js | 2 +- src/bridge/MatrixHandler.js | 2 +- src/irc/ClientPool.js | 2 +- src/irc/IrcEventBroker.js | 2 +- src/models/BridgeRequest.js | 28 ---------------------------- src/provisioning/Provisioner.js | 2 +- 8 files changed, 7 insertions(+), 35 deletions(-) delete mode 100644 src/models/BridgeRequest.js diff --git a/src/DebugApi.js b/src/DebugApi.js index 0942c630c..311283eae 100644 --- a/src/DebugApi.js +++ b/src/DebugApi.js @@ -2,7 +2,7 @@ "use strict"; const querystring = require("querystring"); const Promise = require("bluebird"); -const BridgeRequest = require("./models/BridgeRequest"); +const { BridgeRequest } = require("./models/BridgeRequest"); const log = require("./logging").get("DebugApi"); const http = require("http"); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index e51c1888e..a4ebe4271 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -16,7 +16,7 @@ var BridgedClient = require("../irc/BridgedClient"); var IrcUser = require("../models/IrcUser"); const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); -var BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); var stats = require("../config/stats"); const { NeDBDataStore } = require("../datastore/NedbDataStore"); const { PgDataStore } = require("../datastore/postgres/PgDataStore"); diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index 0dfbea8a0..49fc69e1f 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -2,7 +2,7 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index 3141cc793..a8a7d8cbc 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -9,7 +9,7 @@ const MatrixAction = require("../models/MatrixAction"); const IrcAction = require("../models/IrcAction"); const { IrcClientConfig } = require("../models/IrcClientConfig"); const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const toIrcLowerCase = require("../irc/formatting").toIrcLowerCase; const StateLookup = require('matrix-appservice-bridge').StateLookup; diff --git a/src/irc/ClientPool.js b/src/irc/ClientPool.js index 4858dc816..4220f829d 100644 --- a/src/irc/ClientPool.js +++ b/src/irc/ClientPool.js @@ -8,7 +8,7 @@ const stats = require("../config/stats"); const log = require("../logging").get("ClientPool"); const Promise = require("bluebird"); const QueuePool = require("../util/QueuePool"); -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); class ClientPool { constructor(ircBridge) { diff --git a/src/irc/IrcEventBroker.js b/src/irc/IrcEventBroker.js index 4fe08c8c4..983a87d94 100644 --- a/src/irc/IrcEventBroker.js +++ b/src/irc/IrcEventBroker.js @@ -69,7 +69,7 @@ "use strict"; const IrcAction = require("../models/IrcAction"); const IrcUser = require("../models/IrcUser"); -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const log = require("../logging").get("IrcEventBroker"); const CLEANUP_TIME_MS = 1000 * 60 * 10; // 10min diff --git a/src/models/BridgeRequest.js b/src/models/BridgeRequest.js deleted file mode 100644 index 0e801e34d..000000000 --- a/src/models/BridgeRequest.js +++ /dev/null @@ -1,28 +0,0 @@ -"use strict"; -const logging = require("../logging"); -var log = logging.get("req"); - -class BridgeRequest { - constructor(req) { - this.req = req; - var isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; - this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); - } - - getPromise() { - return this.req.getPromise(); - } - - resolve(thing) { - this.req.resolve(thing); - } - - reject(err) { - this.req.reject(err); - } -} -BridgeRequest.ERR_VIRTUAL_USER = "virtual-user"; -BridgeRequest.ERR_NOT_MAPPED = "not-mapped"; -BridgeRequest.ERR_DROPPED = "dropped"; - -module.exports = BridgeRequest; diff --git a/src/provisioning/Provisioner.js b/src/provisioning/Provisioner.js index 6a2f545c4..689d1d8cc 100644 --- a/src/provisioning/Provisioner.js +++ b/src/provisioning/Provisioner.js @@ -6,7 +6,7 @@ const IrcAction = require("../models/IrcAction"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const ConfigValidator = require("matrix-appservice-bridge").ConfigValidator; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const BridgeRequest = require("../models/BridgeRequest"); +const { BridgeRequest } = require("../models/BridgeRequest"); const ProvisionRequest = require("./ProvisionRequest"); const log = require("../logging").get("Provisioner"); From ca8395f2c7d53f8ae5547790489437186327192e Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:58:43 +0100 Subject: [PATCH 057/350] Convert models/MatrixAction to Typescript --- spec/unit/MatrixAction.spec.js | 2 +- src/bridge/IrcHandler.js | 2 +- src/bridge/MatrixHandler.js | 16 ++-- src/models/MatrixAction.js | 159 ------------------------------ src/models/MatrixAction.ts | 170 +++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 171 deletions(-) delete mode 100644 src/models/MatrixAction.js create mode 100644 src/models/MatrixAction.ts diff --git a/spec/unit/MatrixAction.spec.js b/spec/unit/MatrixAction.spec.js index 51b5fe453..32e281f52 100644 --- a/spec/unit/MatrixAction.spec.js +++ b/spec/unit/MatrixAction.spec.js @@ -1,5 +1,5 @@ "use strict"; -const MatrixAction = require("../../lib/models/MatrixAction"); +const { MatrixAction } = require("../../lib/models/MatrixAction"); const FakeIntent = { getProfileInfo: (userId) => { diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index 49fc69e1f..f18951e92 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -6,7 +6,7 @@ const { BridgeRequest } = require("../models/BridgeRequest"); const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; -const MatrixAction = require("../models/MatrixAction"); +const { MatrixAction } = require("../models/MatrixAction"); const Queue = require("../util/Queue.js"); const QueuePool = require("../util/QueuePool.js"); const QuitDebouncer = require("./QuitDebouncer.js"); diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index a8a7d8cbc..285f11d64 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -5,8 +5,8 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const { IrcRoom } = require("../models/IrcRoom"); -const MatrixAction = require("../models/MatrixAction"); const IrcAction = require("../models/IrcAction"); +const { MatrixAction } = require("../models/MatrixAction"); const { IrcClientConfig } = require("../models/IrcClientConfig"); const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const { BridgeRequest } = require("../models/BridgeRequest"); @@ -43,6 +43,8 @@ function MatrixHandler(ircBridge, config) { this.metrics = { //domain => {"metricname" => value} }; + // The media URL to use to transform mxc:// URLs when handling m.room.[file|image]s + this._mediaUrl = ircBridge.config.homeserver.media_url || ircBridge.config.homeserver.url; } // ===== Matrix Invite Handling ===== @@ -1229,11 +1231,9 @@ MatrixHandler.prototype._onMessage = Promise.coroutine(function*(req, event) { return BridgeRequest.ERR_VIRTUAL_USER; } - // The media URL to use to transform mxc:// URLs when handling m.room.[file|image]s - let mediaUrl = this.ircBridge.config.homeserver.media_url; let mxAction = MatrixAction.fromEvent( - this.ircBridge.getAppServiceBridge().getClientFactory().getClientAs(), event, mediaUrl + event, this._mediaUrl ); let ircAction = IrcAction.fromMatrixAction(mxAction); let ircRooms = yield this.ircBridge.getStore().getIrcChannelsForRoomId(event.room_id); @@ -1411,9 +1411,6 @@ MatrixHandler.prototype._sendIrcAction = Promise.coroutine( req.log.error("Failed to upload text file ", err); } - // The media URL to use to transform mxc:// URLs when handling m.room.[file|image]s - let mediaUrl = this.ircBridge.config.homeserver.media_url; - // This is true if the upload was a success if (result.content_uri) { // Alter event object so that it is treated as if a file has been uploaded @@ -1422,8 +1419,7 @@ MatrixHandler.prototype._sendIrcAction = Promise.coroutine( event.content.body = "sent a long message: "; // Create a file event to reflect the recent upload - let cli = this.ircBridge.getAppServiceBridge().getClientFactory().getClientAs(); - let mAction = MatrixAction.fromEvent(cli, event, mediaUrl); + let mAction = MatrixAction.fromEvent(event, this._mediaUrl); let bigFileIrcAction = IrcAction.fromMatrixAction(mAction); // Replace "Posted a File with..." @@ -1449,7 +1445,7 @@ MatrixHandler.prototype._sendIrcAction = Promise.coroutine( MatrixAction.fromEvent( this.ircBridge.getAppServiceBridge().getClientFactory().getClientAs(), event, - mediaUrl + this._mediaUrl ) ); diff --git a/src/models/MatrixAction.js b/src/models/MatrixAction.js deleted file mode 100644 index f839ca0d4..000000000 --- a/src/models/MatrixAction.js +++ /dev/null @@ -1,159 +0,0 @@ -/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping -"use strict"; -const ircFormatting = require("../irc/formatting"); -const log = require("../logging").get("MatrixAction"); -const ContentRepo = require("matrix-appservice-bridge").ContentRepo; -const escapeStringRegexp = require('escape-string-regexp'); -const Promise = require("bluebird"); - -const ACTION_TYPES = ["message", "emote", "topic", "notice", "file", "image", "video", "audio"]; -const EVENT_TO_TYPE = { - "m.room.message": "message", - "m.room.topic": "topic" -}; -const MSGTYPE_TO_TYPE = { - "m.emote": "emote", - "m.notice": "notice", - "m.image": "image", - "m.video": "video", - "m.audio": "audio", - "m.file": "file" -}; - -const PILL_MIN_LENGTH_TO_MATCH = 4; -const MAX_MATCHES = 5; - -function MatrixAction(type, text, htmlText, timestamp) { - if (ACTION_TYPES.indexOf(type) === -1) { - throw new Error("Unknown MatrixAction type: " + type); - } - this.type = type; - this.text = text; - this.htmlText = htmlText; - this.ts = timestamp || 0; -} - -MatrixAction.prototype.formatMentions = Promise.coroutine(function*(nickUserIdMap, intent) { - const regexString = "(" + - Object.keys(nickUserIdMap).map((value) => escapeStringRegexp(value)).join("|") - + ")"; - const usersRegex = MentionRegex(regexString); - const matched = new Set(); // lowercased nicknames we have matched already. - let match; - for (let i = 0; i < MAX_MATCHES && (match = usersRegex.exec(this.text)) !== null; i++) { - let matchName = match[2]; - // Deliberately have a minimum length to match on, - // so we don't match smaller nicks accidentally. - if (matchName.length < PILL_MIN_LENGTH_TO_MATCH || matched.has(matchName.toLowerCase())) { - continue; - } - let userId = nickUserIdMap[matchName]; - if (userId === undefined) { - // We might need to search case-insensitive. - const nick = Object.keys(nickUserIdMap).find((n) => - n.toLowerCase() === matchName.toLowerCase() - ); - if (nick === undefined) { - continue; - } - userId = nickUserIdMap[nick]; - matchName = nick; - } - // If this message is not HTML, we should make it so. - if (this.htmlText === undefined) { - // This looks scary and unsafe, but further down we check - // if `text` contains any HTML and escape + set `htmlText` appropriately. - this.htmlText = this.text; - } - userId = ircFormatting.escapeHtmlChars(userId); - - /* Due to how Riot and friends do push notifications, - we need the plain text to match something.*/ - let identifier; - try { - identifier = (yield intent.getProfileInfo(userId, 'displayname', true)).displayname; - } - catch (e) { - // This shouldn't happen, but let's not fail to match if so. - } - - if (identifier === undefined) { - // Fallback to userid. - identifier = userId.substr(1, userId.indexOf(":")-1) - } - - const regex = MentionRegex(escapeStringRegexp(matchName)); - this.htmlText = this.htmlText.replace(regex, - `$1`+ - `${ircFormatting.escapeHtmlChars(identifier)}` - ); - this.text = this.text.replace(regex, `$1${identifier}`); - // Don't match this name twice, we've already replaced all entries. - matched.add(matchName.toLowerCase()); - } -}); - -MatrixAction.fromEvent = function(client, event, mediaUrl) { - event.content = event.content || {}; - let type = EVENT_TO_TYPE[event.type] || "message"; // mx event type to action type - let text = event.content.body; - let htmlText = null; - - if (event.type === "m.room.topic") { - text = event.content.topic; - } - else if (event.type === "m.room.message") { - if (event.content.format === "org.matrix.custom.html") { - htmlText = event.content.formatted_body; - } - if (MSGTYPE_TO_TYPE[event.content.msgtype]) { - type = MSGTYPE_TO_TYPE[event.content.msgtype]; - } - if (["m.image", "m.file", "m.video", "m.audio"].indexOf(event.content.msgtype) !== -1) { - var fileSize = ""; - if (event.content.info && event.content.info.size && - typeof event.content.info.size === "number") { - fileSize = " (" + Math.round(event.content.info.size / 1024) + "KB)"; - } - - // By default assume that the media server = client homeserver - if (!mediaUrl) { - mediaUrl = client.getHomeserverUrl(); - } - - const url = ContentRepo.getHttpUriForMxc(mediaUrl, event.content.url); - text = `${event.content.body}${fileSize} < ${url} >`; - } - } - return new MatrixAction(type, text, htmlText, event.origin_server_ts); -}; -MatrixAction.fromIrcAction = function(ircAction) { - switch (ircAction.type) { - case "message": - case "emote": - case "notice": - let htmlText = ircFormatting.ircToHtml(ircAction.text); - return new MatrixAction( - ircAction.type, - ircFormatting.stripIrcFormatting(ircAction.text), - // only set HTML text if we think there is HTML, else the bridge - // will send everything as HTML and never text only. - ircAction.text !== htmlText ? htmlText : undefined - ); - case "topic": - return new MatrixAction("topic", ircAction.text); - default: - log.error("MatrixAction.fromIrcAction: Unknown action: %s", ircAction.type); - return null; - } -}; - -function MentionRegex(matcher) { - const WORD_BOUNDARY = "^|\:|\#|```|\\s|$|,"; - return new RegExp( - `(${WORD_BOUNDARY})(@?(${matcher}))(?=${WORD_BOUNDARY})`, - "igmu" - ); -} - -module.exports = MatrixAction; diff --git a/src/models/MatrixAction.ts b/src/models/MatrixAction.ts new file mode 100644 index 000000000..5eb4ad80e --- /dev/null +++ b/src/models/MatrixAction.ts @@ -0,0 +1,170 @@ +/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping +"use strict"; + +import { IrcAction } from "./IrcAction"; + +const ircFormatting = require("../irc/formatting"); +const log = require("../logging").get("MatrixAction"); +const ContentRepo = require("matrix-appservice-bridge").ContentRepo; +const escapeStringRegexp = require('escape-string-regexp'); + +const ACTION_TYPES = ["message", "emote", "topic", "notice", "file", "image", "video", "audio"]; +const EVENT_TO_TYPE: {[mxKey: string]: string} = { + "m.room.message": "message", + "m.room.topic": "topic" +}; +const MSGTYPE_TO_TYPE: {[mxKey: string]: string} = { + "m.emote": "emote", + "m.notice": "notice", + "m.image": "image", + "m.video": "video", + "m.audio": "audio", + "m.file": "file" +}; + +const PILL_MIN_LENGTH_TO_MATCH = 4; +const MAX_MATCHES = 5; + +interface MatrixEventContent { + body: string; + topic: string; + format: string; + formatted_body: string; + msgtype: string; + url: string; + info?: { + size: number; + } +} + +export class MatrixAction { + + constructor( + public readonly type: string, + public text: string, + public htmlText: string|null = null, + public readonly ts: number = 0 + ) { + if (ACTION_TYPES.indexOf(type) === -1) { + throw new Error("Unknown MatrixAction type: " + type); + } + } + + public async formatMentions(nickUserIdMap: {[nick: string]: string}, intent: any) { + const regexString = `(${Object.keys(nickUserIdMap).map((value) => escapeStringRegexp(value)).join("|")})`; + const usersRegex = MentionRegex(regexString); + const matched = new Set(); // lowercased nicknames we have matched already. + let match; + for (let i = 0; i < MAX_MATCHES && (match = usersRegex.exec(this.text)) !== null; i++) { + let matchName = match[2]; + // Deliberately have a minimum length to match on, + // so we don't match smaller nicks accidentally. + if (matchName.length < PILL_MIN_LENGTH_TO_MATCH || matched.has(matchName.toLowerCase())) { + continue; + } + let userId = nickUserIdMap[matchName]; + if (userId === undefined) { + // We might need to search case-insensitive. + const nick = Object.keys(nickUserIdMap).find((n) => + n.toLowerCase() === matchName.toLowerCase() + ); + if (nick === undefined) { + continue; + } + userId = nickUserIdMap[nick]; + matchName = nick; + } + // If this message is not HTML, we should make it so. + if (!this.htmlText) { + // This looks scary and unsafe, but further down we check + // if `text` contains any HTML and escape + set `htmlText` appropriately. + this.htmlText = this.text; + } + userId = ircFormatting.escapeHtmlChars(userId); + + /* Due to how Riot and friends do push notifications, + we need the plain text to match something.*/ + let identifier; + try { + identifier = (await intent.getProfileInfo(userId, 'displayname', true)).displayname; + } + catch (e) { + // This shouldn't happen, but let's not fail to match if so. + } + + if (identifier === undefined) { + // Fallback to userid. + identifier = userId.substr(1, userId.indexOf(":")-1) + } + + const regex = MentionRegex(escapeStringRegexp(matchName)); + this.htmlText = this.htmlText.replace(regex, + `$1`+ + `${ircFormatting.escapeHtmlChars(identifier)}` + ); + this.text = this.text.replace(regex, `$1${identifier}`); + // Don't match this name twice, we've already replaced all entries. + matched.add(matchName.toLowerCase()); + } + } + + public static fromEvent(event: {type: string, content: MatrixEventContent, origin_server_ts: number}, mediaUrl: string) { + event.content = event.content || {}; + let type = EVENT_TO_TYPE[event.type] || "message"; // mx event type to action type + let text = event.content.body; + let htmlText = null; + + if (event.type === "m.room.topic") { + text = event.content.topic; + } + else if (event.type === "m.room.message") { + if (event.content.format === "org.matrix.custom.html") { + htmlText = event.content.formatted_body; + } + if (MSGTYPE_TO_TYPE[event.content.msgtype]) { + type = MSGTYPE_TO_TYPE[event.content.msgtype]; + } + if (["m.image", "m.file", "m.video", "m.audio"].indexOf(event.content.msgtype) !== -1) { + var fileSize = ""; + if (event.content.info && event.content.info.size && + typeof event.content.info.size === "number") { + fileSize = " (" + Math.round(event.content.info.size / 1024) + "KB)"; + } + + const url = ContentRepo.getHttpUriForMxc(mediaUrl, event.content.url); + text = `${event.content.body}${fileSize} < ${url} >`; + } + } + return new MatrixAction(type, text, htmlText, event.origin_server_ts); + } + + public static fromIrcAction(ircAction: IrcAction) { + switch (ircAction.type) { + case "message": + case "emote": + case "notice": + let htmlText = ircFormatting.ircToHtml(ircAction.text); + return new MatrixAction( + ircAction.type, + ircFormatting.stripIrcFormatting(ircAction.text), + // only set HTML text if we think there is HTML, else the bridge + // will send everything as HTML and never text only. + ircAction.text !== htmlText ? htmlText : undefined + ); + case "topic": + return new MatrixAction("topic", ircAction.text); + default: + log.error("MatrixAction.fromIrcAction: Unknown action: %s", ircAction.type); + return null; + } + } +} + + +function MentionRegex(matcher: string) { + const WORD_BOUNDARY = "^|\:|\#|```|\\s|$|,"; + return new RegExp( + `(${WORD_BOUNDARY})(@?(${matcher}))(?=${WORD_BOUNDARY})`, + "igmu" + ); +} \ No newline at end of file From c4799dcda099aad4724ff266d5629755af20594e Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:59:06 +0100 Subject: [PATCH 058/350] Convert models/IrcAction to Typescript --- src/bridge/MatrixHandler.js | 2 +- src/models/IrcAction.js | 47 -------------------------- src/models/IrcAction.ts | 67 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 48 deletions(-) delete mode 100644 src/models/IrcAction.js create mode 100644 src/models/IrcAction.ts diff --git a/src/bridge/MatrixHandler.js b/src/bridge/MatrixHandler.js index 285f11d64..1d280800d 100644 --- a/src/bridge/MatrixHandler.js +++ b/src/bridge/MatrixHandler.js @@ -5,8 +5,8 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const { IrcRoom } = require("../models/IrcRoom"); -const IrcAction = require("../models/IrcAction"); const { MatrixAction } = require("../models/MatrixAction"); +const { IrcAction } = require("../models/IrcAction"); const { IrcClientConfig } = require("../models/IrcClientConfig"); const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const { BridgeRequest } = require("../models/BridgeRequest"); diff --git a/src/models/IrcAction.js b/src/models/IrcAction.js deleted file mode 100644 index f66eb084b..000000000 --- a/src/models/IrcAction.js +++ /dev/null @@ -1,47 +0,0 @@ -"use strict"; -const ircFormatting = require("../irc/formatting"); -const log = require("../logging").get("IrcAction"); - -const ACTION_TYPES = ["message", "emote", "topic", "notice"]; - -function IrcAction(type, text, timestamp) { - if (ACTION_TYPES.indexOf(type) === -1) { - throw new Error("Unknown IrcAction type: " + type); - } - this.type = type; - this.text = text; - this.ts = timestamp || 0; -} -IrcAction.fromMatrixAction = function(matrixAction) { - switch (matrixAction.type) { - case "message": - case "emote": - case "notice": - if (matrixAction.htmlText) { - let ircText = ircFormatting.htmlToIrc(matrixAction.htmlText); - if (ircText === null) { - ircText = matrixAction.text; // fallback - } - // irc formatted text is the main text part - return new IrcAction(matrixAction.type, ircText, matrixAction.ts) - } - return new IrcAction(matrixAction.type, matrixAction.text, matrixAction.ts); - case "image": - return new IrcAction( - "emote", "uploaded an image: " + matrixAction.text, matrixAction.ts - ); - case "video": - return new IrcAction( - "emote", "uploaded a video: " + matrixAction.text, matrixAction.ts - ); - case "file": - return new IrcAction("emote", "posted a file: " + matrixAction.text, matrixAction.ts); - case "topic": - return new IrcAction(matrixAction.type, matrixAction.text, matrixAction.ts); - default: - log.error("IrcAction.fromMatrixAction: Unknown action: %s", matrixAction.type); - return null; - } -}; - -module.exports = IrcAction; diff --git a/src/models/IrcAction.ts b/src/models/IrcAction.ts new file mode 100644 index 000000000..20f7bd26e --- /dev/null +++ b/src/models/IrcAction.ts @@ -0,0 +1,67 @@ + +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const ircFormatting = require("../irc/formatting"); +const log = require("../logging").get("IrcAction"); + +import { MatrixAction } from "./MatrixAction"; + +const ACTION_TYPES = ["message", "emote", "topic", "notice"]; +type IrcActionType = "message"|"emote"|"topic"|"notice"; + +export class IrcAction { + constructor ( + public readonly type: IrcActionType, + public readonly text: string, + public readonly ts: number = 0 ) { + if (ACTION_TYPES.indexOf(type) === -1) { + throw new Error("Unknown IrcAction type: " + type); + } + } + + public static fromMatrixAction(matrixAction: MatrixAction): IrcAction|null { + switch (matrixAction.type) { + case "message": + case "emote": + case "notice": + if (matrixAction.htmlText) { + let ircText = ircFormatting.htmlToIrc(matrixAction.htmlText); + if (ircText === null) { + ircText = matrixAction.text; // fallback + } + // irc formatted text is the main text part + return new IrcAction(matrixAction.type, ircText, matrixAction.ts) + } + return new IrcAction(matrixAction.type, matrixAction.text, matrixAction.ts); + case "image": + return new IrcAction( + "emote", "uploaded an image: " + matrixAction.text, matrixAction.ts + ); + case "video": + return new IrcAction( + "emote", "uploaded a video: " + matrixAction.text, matrixAction.ts + ); + case "file": + return new IrcAction("emote", "posted a file: " + matrixAction.text, matrixAction.ts); + case "topic": + return new IrcAction(matrixAction.type, matrixAction.text, matrixAction.ts); + default: + log.error("IrcAction.fromMatrixAction: Unknown action: %s", matrixAction.type); + return null; + } + } +} From c9600cb05dedb5573c2cfa6744a78d600120baa9 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 16:59:39 +0100 Subject: [PATCH 059/350] Add BridgeRequest.ts --- src/models/BridgeRequest.ts | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/models/BridgeRequest.ts diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts new file mode 100644 index 000000000..4999ca6bd --- /dev/null +++ b/src/models/BridgeRequest.ts @@ -0,0 +1,53 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const logging = require("../logging"); +const log = logging.get("req"); + +interface Req { + getPromise(): Promise; + resolve(thing: T): void; + reject(err: T): void; + getId(): string; + getData(): { + isFromIrc: boolean; + } | undefined +} + +export class BridgeRequest { + public readonly log: any; + constructor(public readonly req: Req) { + const data = req.getData(); + const isFromIrc = data ? Boolean(data.isFromIrc) : false; + this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); + } + + public getPromise(): Promise { + return this.req.getPromise(); + } + + public resolve(thing: T) { + this.req.resolve(thing); + } + + public reject(err: T) { + this.req.reject(err); + } + + public static readonly ERR_VIRTUAL_USER = "virtual-user"; + public static readonly ERR_NOT_MAPPED = "not-mapped"; + public static readonly ERR_DROPPED = "dropped"; +} From 3660146f87d25a2f965c4776aea877a1da006210 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 17:09:02 +0100 Subject: [PATCH 060/350] Convert models/IrcUser to Typescript --- src/models/IrcClientConfig.ts | 2 -- src/models/IrcRoom.ts | 1 - src/models/IrcUser.js | 47 ----------------------------------- src/models/IrcUser.ts | 39 +++++++++++++++++++++++++++++ src/models/MatrixAction.ts | 17 +++++++++++-- 5 files changed, 54 insertions(+), 52 deletions(-) delete mode 100644 src/models/IrcUser.js create mode 100644 src/models/IrcUser.ts diff --git a/src/models/IrcClientConfig.ts b/src/models/IrcClientConfig.ts index bc2883942..3f712c12c 100644 --- a/src/models/IrcClientConfig.ts +++ b/src/models/IrcClientConfig.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Ignore definition errors for now. -//@ts-ignore import { MatrixUser } from "matrix-appservice-bridge"; export interface IrcClientConfigSeralized { diff --git a/src/models/IrcRoom.ts b/src/models/IrcRoom.ts index e51ac7e18..7db323000 100644 --- a/src/models/IrcRoom.ts +++ b/src/models/IrcRoom.ts @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -//@ts-ignore import { RemoteRoom } from "matrix-appservice-bridge"; import { toIrcLowerCase } from "../irc/formatting"; import { IrcServer } from "../irc/IrcServer"; diff --git a/src/models/IrcUser.js b/src/models/IrcUser.js deleted file mode 100644 index c8cf48ef2..000000000 --- a/src/models/IrcUser.js +++ /dev/null @@ -1,47 +0,0 @@ -"use strict"; -const RemoteUser = require("matrix-appservice-bridge").RemoteUser; - -class IrcUser extends RemoteUser { - - /** - * Construct a new IRC user. - * @constructor - * @param {IrcServer} server : The IRC server the user is on. - * @param {string} nick : The nick for this user. - * @param {boolean} isVirtual : True if the user is not a real IRC user. - * @param {string} password : The password to give to NickServ. - * @param {string} username : The username of the client (for ident) - */ - constructor(server, nick, isVirtual, password, username) { - super(server.domain + "__@__" + nick, { - domain: server.domain, - nick: nick, - isVirtual: Boolean(isVirtual), - password: password || null, - username: username || null - }); - this.isVirtual = Boolean(isVirtual); - this.server = server; - this.nick = nick; - this.password = password || null; - } - - getUsername() { - return this.get("username"); - } - - toString() { - return this.nick + " (" + this.getUsername() + "@" + - (this.server ? this.server.domain : "-") + ")"; - } -} - -IrcUser.fromRemoteUser = function(server, remoteUser) { - var ircUser = new IrcUser( - server, remoteUser.get("nick"), remoteUser.get("isVirtual"), - remoteUser.get("password"), remoteUser.get("username") - ); - return ircUser; -}; - -module.exports = IrcUser; diff --git a/src/models/IrcUser.ts b/src/models/IrcUser.ts new file mode 100644 index 000000000..a460adfd7 --- /dev/null +++ b/src/models/IrcUser.ts @@ -0,0 +1,39 @@ +"use strict"; + +import { RemoteUser } from "matrix-appservice-bridge"; +import { IrcServer } from "../irc/IrcServer"; + +export class IrcUser extends RemoteUser { + + /** + * Construct a new IRC user. + * @constructor + * @param {IrcServer} server : The IRC server the user is on. + * @param {string} nick : The nick for this user. + * @param {boolean} isVirtual : True if the user is not a real IRC user. + * @param {string} password : The password to give to NickServ. + * @param {string} username : The username of the client (for ident) + */ + constructor( + public readonly server: IrcServer, + public readonly nick: string, + public readonly isVirtual: boolean, + public readonly password: string|null = null, + username: string|null = null) { + super(server.domain + "__@__" + nick, { + domain: server.domain, + nick: nick, + isVirtual: Boolean(isVirtual), + password: password || null, + username: username || null + }); + } + + getUsername(): string { + return this.get("username") as string; + } + + toString() { + return `${this.nick} (${this.getUsername()}@${this.server ? this.server.domain : "-"})`; + } +} diff --git a/src/models/MatrixAction.ts b/src/models/MatrixAction.ts index 5eb4ad80e..21ab99847 100644 --- a/src/models/MatrixAction.ts +++ b/src/models/MatrixAction.ts @@ -1,5 +1,18 @@ -/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping -"use strict"; +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ import { IrcAction } from "./IrcAction"; From bbefdb6796887c7d407be58116c3a0257b623875 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 17:55:03 +0100 Subject: [PATCH 061/350] Test and import tweaks --- spec/integ/matrix-to-irc.spec.js | 18 ++++++------------ spec/unit/MatrixAction.spec.js | 2 +- src/bridge/IrcBridge.js | 2 +- src/irc/IrcEventBroker.js | 4 ++-- src/provisioning/Provisioner.js | 2 +- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/spec/integ/matrix-to-irc.spec.js b/spec/integ/matrix-to-irc.spec.js index a9292fb0f..a39dba258 100644 --- a/spec/integ/matrix-to-irc.spec.js +++ b/spec/integ/matrix-to-irc.spec.js @@ -7,7 +7,7 @@ const mediaUrl = "http://some-media-repo.com"; describe("Matrix-to-IRC message bridging", function() { - const {env, config, roomMapping, test} = envBundle(); + const {env, roomMapping, test} = envBundle(); let testUser = { id: "@flibble:wibble", @@ -472,10 +472,7 @@ describe("Matrix-to-IRC message bridging", function() { it("should bridge matrix images as IRC action with a URL", function(done) { let tBody = "the_image.jpg"; let tMxcSegment = "/somecontentid"; - let tHsUrl = "http://somedomain.com"; - let sdk = env.clientMock._client(config._botUserId); - - sdk.getHomeserverUrl.and.returnValue(tHsUrl); + let tHsUrl = "https://some.home.server.goeshere/"; env.ircMock._whenClient(roomMapping.server, testUser.nick, "action", function(client, channel, text) { @@ -505,10 +502,7 @@ describe("Matrix-to-IRC message bridging", function() { it("should bridge matrix files as IRC action with a URL", function(done) { let tBody = "a_file.apk"; let tMxcSegment = "/somecontentid"; - let tHsUrl = "http://somedomain.com"; - let sdk = env.clientMock._client(config._botUserId); - - sdk.getHomeserverUrl.and.returnValue(tHsUrl); + let tHsUrl = "https://some.home.server.goeshere/"; env.ircMock._whenClient(roomMapping.server, testUser.nick, "action", function(client, channel, text) { @@ -730,8 +724,6 @@ describe("Matrix-to-IRC message bridging with media URL and drop time", function }; beforeEach(test.coroutine(function*() { - // Set the media URL - env.config.homeserver.media_url = mediaUrl; env.config.homeserver.dropMatrixMessagesAfterSecs = 300; // 5 min jasmine.clock().install(); @@ -753,6 +745,8 @@ describe("Matrix-to-IRC message bridging with media URL and drop time", function // do the init yield test.initEnv(env); + // Set the media URL + env.ircBridge.matrixHandler._mediaUrl = mediaUrl; })); afterEach(test.coroutine(function*() { @@ -860,7 +854,7 @@ describe("Matrix-to-IRC message bridging with media URL and drop time", function let tHsUrl = "http://somedomain.com"; let sdk = env.clientMock._client(config._botUserId); - // Not expected to be caleld, but hook to catch the error + // Not expected to be called, but hook to catch the error // see expectation not to see HS URL, below sdk.getHomeserverUrl.and.returnValue(tHsUrl); diff --git a/spec/unit/MatrixAction.spec.js b/spec/unit/MatrixAction.spec.js index 32e281f52..13e04c3f7 100644 --- a/spec/unit/MatrixAction.spec.js +++ b/spec/unit/MatrixAction.spec.js @@ -25,7 +25,7 @@ describe("MatrixAction", function() { "Some Person": "@foobar:localhost" }, FakeIntent).then(() => { expect(action.text).toEqual("Some text"); - expect(action.htmlText).toBeUndefined(); + expect(action.htmlText).toBeNull(); }); }); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index a4ebe4271..57a4463cf 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -13,7 +13,7 @@ const { IrcServer } = require("../irc/IrcServer.js"); var ClientPool = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); var BridgedClient = require("../irc/BridgedClient"); -var IrcUser = require("../models/IrcUser"); +const { IrcUser } = require("../models/IrcUser"); const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); const { BridgeRequest } = require("../models/BridgeRequest"); diff --git a/src/irc/IrcEventBroker.js b/src/irc/IrcEventBroker.js index 983a87d94..5d734d81c 100644 --- a/src/irc/IrcEventBroker.js +++ b/src/irc/IrcEventBroker.js @@ -67,8 +67,8 @@ */ "use strict"; -const IrcAction = require("../models/IrcAction"); -const IrcUser = require("../models/IrcUser"); +const { IrcAction } = require("../models/IrcAction"); +const { IrcUser } = require("../models/IrcUser"); const { BridgeRequest } = require("../models/BridgeRequest"); const log = require("../logging").get("IrcEventBroker"); diff --git a/src/provisioning/Provisioner.js b/src/provisioning/Provisioner.js index 689d1d8cc..3b816107f 100644 --- a/src/provisioning/Provisioner.js +++ b/src/provisioning/Provisioner.js @@ -2,7 +2,7 @@ "use strict"; const Promise = require("bluebird"); const { IrcRoom } = require("../models/IrcRoom"); -const IrcAction = require("../models/IrcAction"); +const { IrcAction } = require("../models/IrcAction"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const ConfigValidator = require("matrix-appservice-bridge").ConfigValidator; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; From aae07f9b7f3fa943f952f66d66c0275fbf095dca Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 18:10:54 +0100 Subject: [PATCH 062/350] Linting --- package-lock.json | 26 ++++++++-- package.json | 2 +- src/models/BridgeRequest.ts | 6 ++- src/models/IrcAction.ts | 2 +- src/models/IrcUser.ts | 2 +- src/models/MatrixAction.ts | 59 ++++++++++++----------- types/matrix-appservice-bridge/index.d.ts | 8 +++ 7 files changed, 69 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 95d552800..02556057a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -614,6 +614,13 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + } } }, "chardet": { @@ -1068,9 +1075,9 @@ "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==" }, "eslint": { "version": "5.16.0", @@ -1344,6 +1351,14 @@ "dev": true, "requires": { "escape-string-regexp": "^1.0.5" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + } } }, "file-entry-cache": { @@ -2574,6 +2589,11 @@ "ms": "^2.1.1" } }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, "glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", diff --git a/package.json b/package.json index 9441b498e..3e0b4615e 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "bluebird": "^3.1.1", "crc": "^3.2.1", - "escape-string-regexp": "^1.0.5", + "escape-string-regexp": "^2.0.0", "extend": "^2.0.0", "he": "^1.1.1", "iconv": "^2.3.4", diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index 4999ca6bd..46d79cb7b 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -const logging = require("../logging"); +import logging = require("../logging"); const log = logging.get("req"); interface Req { @@ -24,10 +24,12 @@ interface Req { getId(): string; getData(): { isFromIrc: boolean; - } | undefined + } | undefined; } export class BridgeRequest { + // We don't have a type for this yet + // eslint-disable-next-line @typescript-eslint/no-explicit-any public readonly log: any; constructor(public readonly req: Req) { const data = req.getData(); diff --git a/src/models/IrcAction.ts b/src/models/IrcAction.ts index 20f7bd26e..b8faf37a4 100644 --- a/src/models/IrcAction.ts +++ b/src/models/IrcAction.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -const ircFormatting = require("../irc/formatting"); +import ircFormatting = require("../irc/formatting"); const log = require("../logging").get("IrcAction"); import { MatrixAction } from "./MatrixAction"; diff --git a/src/models/IrcUser.ts b/src/models/IrcUser.ts index a460adfd7..df73ac015 100644 --- a/src/models/IrcUser.ts +++ b/src/models/IrcUser.ts @@ -17,7 +17,7 @@ export class IrcUser extends RemoteUser { constructor( public readonly server: IrcServer, public readonly nick: string, - public readonly isVirtual: boolean, + public readonly isVirtual: boolean, public readonly password: string|null = null, username: string|null = null) { super(server.domain + "__@__" + nick, { diff --git a/src/models/MatrixAction.ts b/src/models/MatrixAction.ts index 21ab99847..2e0dc3223 100644 --- a/src/models/MatrixAction.ts +++ b/src/models/MatrixAction.ts @@ -16,10 +16,10 @@ limitations under the License. import { IrcAction } from "./IrcAction"; -const ircFormatting = require("../irc/formatting"); +import ircFormatting = require("../irc/formatting"); const log = require("../logging").get("MatrixAction"); -const ContentRepo = require("matrix-appservice-bridge").ContentRepo; -const escapeStringRegexp = require('escape-string-regexp'); +import { ContentRepo, Intent } from "matrix-appservice-bridge"; +import escapeStringRegexp from "escape-string-regexp"; const ACTION_TYPES = ["message", "emote", "topic", "notice", "file", "image", "video", "audio"]; const EVENT_TO_TYPE: {[mxKey: string]: string} = { @@ -38,16 +38,28 @@ const MSGTYPE_TO_TYPE: {[mxKey: string]: string} = { const PILL_MIN_LENGTH_TO_MATCH = 4; const MAX_MATCHES = 5; -interface MatrixEventContent { - body: string; - topic: string; - format: string; - formatted_body: string; - msgtype: string; - url: string; - info?: { - size: number; - } +interface MatrixEvent { + type: string; + content: { + body: string; + topic: string; + format: string; + formatted_body: string; + msgtype: string; + url: string; + info?: { + size: number; + }; + }; + origin_server_ts: number; +} + +const MentionRegex = function(matcher: string): RegExp { + const WORD_BOUNDARY = "^|\:|\#|```|\\s|$|,"; + return new RegExp( + `(${WORD_BOUNDARY})(@?(${matcher}))(?=${WORD_BOUNDARY})`, + "igmu" + ); } export class MatrixAction { @@ -63,7 +75,7 @@ export class MatrixAction { } } - public async formatMentions(nickUserIdMap: {[nick: string]: string}, intent: any) { + public async formatMentions(nickUserIdMap: {[nick: string]: string}, intent: Intent) { const regexString = `(${Object.keys(nickUserIdMap).map((value) => escapeStringRegexp(value)).join("|")})`; const usersRegex = MentionRegex(regexString); const matched = new Set(); // lowercased nicknames we have matched already. @@ -121,12 +133,12 @@ export class MatrixAction { } } - public static fromEvent(event: {type: string, content: MatrixEventContent, origin_server_ts: number}, mediaUrl: string) { + public static fromEvent(event: MatrixEvent, mediaUrl: string) { event.content = event.content || {}; let type = EVENT_TO_TYPE[event.type] || "message"; // mx event type to action type let text = event.content.body; let htmlText = null; - + if (event.type === "m.room.topic") { text = event.content.topic; } @@ -138,12 +150,12 @@ export class MatrixAction { type = MSGTYPE_TO_TYPE[event.content.msgtype]; } if (["m.image", "m.file", "m.video", "m.audio"].indexOf(event.content.msgtype) !== -1) { - var fileSize = ""; + let fileSize = ""; if (event.content.info && event.content.info.size && typeof event.content.info.size === "number") { fileSize = " (" + Math.round(event.content.info.size / 1024) + "KB)"; } - + const url = ContentRepo.getHttpUriForMxc(mediaUrl, event.content.url); text = `${event.content.body}${fileSize} < ${url} >`; } @@ -156,7 +168,7 @@ export class MatrixAction { case "message": case "emote": case "notice": - let htmlText = ircFormatting.ircToHtml(ircAction.text); + const htmlText = ircFormatting.ircToHtml(ircAction.text); return new MatrixAction( ircAction.type, ircFormatting.stripIrcFormatting(ircAction.text), @@ -172,12 +184,3 @@ export class MatrixAction { } } } - - -function MentionRegex(matcher: string) { - const WORD_BOUNDARY = "^|\:|\#|```|\\s|$|,"; - return new RegExp( - `(${WORD_BOUNDARY})(@?(${matcher}))(?=${WORD_BOUNDARY})`, - "igmu" - ); -} \ No newline at end of file diff --git a/types/matrix-appservice-bridge/index.d.ts b/types/matrix-appservice-bridge/index.d.ts index 1ec9e451c..3374b0a75 100644 --- a/types/matrix-appservice-bridge/index.d.ts +++ b/types/matrix-appservice-bridge/index.d.ts @@ -142,4 +142,12 @@ declare module 'matrix-appservice-bridge' { unlinkUserIds (matrixUserId: string, remoteUserId: string): Promise unlinkUsers (matrixUser: MatrixUser, remoteUser: RemoteUser): Promise } + + export class ContentRepo { + static getHttpUriForMxc(baseUrl: string, mxc: string): string; + } + + export class Intent { + getProfileInfo(userId: string, type?: "displayname"|"avatar_url", useCache?: boolean): Promise<{displayname: string|null, avatar_url: string|null}> + } } \ No newline at end of file From 11245e32b36a6aae2926dbae8aa18fa70ff3c620 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 25 Sep 2019 18:14:18 +0100 Subject: [PATCH 063/350] Newsfile --- changelog.d/822.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/822.misc diff --git a/changelog.d/822.misc b/changelog.d/822.misc new file mode 100644 index 000000000..223065ae6 --- /dev/null +++ b/changelog.d/822.misc @@ -0,0 +1 @@ +Internal conversions of model classes to Typescript \ No newline at end of file From 3f14343ba3f44fa48ed3f5712c76eaa61ba09b66 Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Wed, 16 May 2018 18:54:17 +0200 Subject: [PATCH 064/350] Make ident reliable by delaying negative response The IRC server typically sends the ident request as soon as the TCP connection is established. However, since we only add the ident mapping after we get the established connection socket, the response is extremely racy and often fails to find the mapping. This commit follows [the scheme implemented by Quassel][1] and delays giving a negative response while there are still connections in the process of being opened and are older than the request. This should make the response reliable. [1]: https://github.com/quassel/quassel/pull/356 --- src/irc/BridgedClient.js | 9 ++++++-- src/irc/ident.js | 45 +++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/irc/BridgedClient.js b/src/irc/BridgedClient.js index 3dafe6a04..da55cad51 100644 --- a/src/irc/BridgedClient.js +++ b/src/irc/BridgedClient.js @@ -127,6 +127,8 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { `Connecting to the IRC network '${this.server.domain}' as ${this.nick}...` ); + let identToken = ident.takeToken(); + let connInst = yield ConnectionInstance.create(server, { nick: this.nick, username: nameInfo.username, @@ -138,7 +140,7 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { this.server.getIpv6Prefix() ? this._clientConfig.getIpv6Address() : undefined ) }, (inst) => { - this._onConnectionCreated(inst, nameInfo); + this._onConnectionCreated(inst, nameInfo, identToken); }); this.inst = connInst; @@ -190,6 +192,7 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { catch (err) { this.log.debug("Failed to connect."); this.instCreationFailed = true; + ident.returnToken(identToken); throw err; } }); @@ -582,7 +585,7 @@ BridgedClient.prototype._addChannel = function(channel) { BridgedClient.prototype.getLastActionTs = function() { return this.lastActionTs; }; -BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo) { +BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, identToken) { // listen for a connect event which is done when the TCP connection is // established and set ident info (this is different to the connect() callback // in node-irc which actually fires on a registered event..) @@ -594,6 +597,7 @@ BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo) { if (localPort > 0) { ident.setMapping(nameInfo.username, localPort); } + ident.returnToken(identToken); }); connInst.onDisconnect = (reason) => { @@ -608,6 +612,7 @@ BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo) { "' has been lost. " ); clearTimeout(this._idleTimeout); + ident.returnToken(identToken); }; this._eventBroker.addHooks(this, connInst); diff --git a/src/irc/ident.js b/src/irc/ident.js index 075663bba..d25b42c45 100644 --- a/src/irc/ident.js +++ b/src/irc/ident.js @@ -15,6 +15,7 @@ */ "use strict"; +const EventEmitter = require('events'); const net = require('net'); const log = require("../logging").get("irc-ident"); @@ -27,6 +28,10 @@ var portMappings = { // port: username }; +var nextToken = 1; +const openTokens = []; +const emitter = new EventEmitter(); + var respond = function(sock, localPort, remotePort, username) { var response; if (username) { @@ -41,6 +46,21 @@ var respond = function(sock, localPort, remotePort, username) { sock.end(response); }; +const tryRespond = (currentToken, sock, localPort, remotePort) => { + let username = portMappings[localPort]; + if (username) { + log.debug("Port %s is %s", localPort, username); + respond(sock, localPort, remotePort, username); + } + else if (openTokens.length > 0 && openTokens[0] < currentToken) { + emitter.once("token", () => tryRespond(currentToken, sock, localPort, remotePort)); + } + else { + log.debug("No user on port %s", localPort); + respond(sock, localPort, remotePort, null); + } +}; + module.exports = { configure: function(opts) { log.info("Configuring ident server => %s", JSON.stringify(opts)); @@ -58,14 +78,9 @@ module.exports = { log.debug("BAD DATA"); return; } - var username = portMappings[String(localOutgoingPort)]; - if (!username) { - log.debug("No user on port %s", localOutgoingPort); - respond(sock, localOutgoingPort, remoteConnectPort, null); - return; - } - log.debug("Port %s is %s", localOutgoingPort, username); - respond(sock, localOutgoingPort, remoteConnectPort, username); + tryRespond(nextToken, sock, + String(localOutgoingPort), + String(remoteConnectPort)); }); sock.on("close", function() { log.debug("CLOSE"); @@ -91,5 +106,19 @@ module.exports = { } }); } + }, + takeToken: function() { + let token = nextToken++; + openTokens.push(token); + log.debug("Took token %d", token); + return token; + }, + returnToken: function(token) { + let index = openTokens.indexOf(token); + if (index > -1) { + openTokens.splice(index, 1); + log.debug("Returned token %d", token); + emitter.emit('token'); + } } }; From ea088727234230f46ba69038c11fa6d3674b4e8f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 12:47:50 +0100 Subject: [PATCH 065/350] Support room upgrades on Postgres --- src/bridge/IrcBridge.js | 10 ++++++++-- src/datastore/DataStore.ts | 2 ++ src/datastore/NedbDataStore.ts | 4 ++++ src/datastore/postgres/PgDataStore.ts | 4 ++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index e51c1888e..c7837a214 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -93,6 +93,11 @@ function IrcBridge(config, registration) { userStore: `${dirPath}/users.db`, }; } + else { + bridgeStoreConfig = { + disableStores: true, + }; + } this._bridge = new Bridge({ registration: this.registration, @@ -320,8 +325,9 @@ IrcBridge.prototype.createBridgedClient = function(ircClientConfig, matrixUser, IrcBridge.prototype.run = Promise.coroutine(function*(port) { yield this._bridge.loadDatabases(); + const dbConfig = this.config.database; - if (this._debugApi) { + if (this._debugApi && dbConfig.engine === "nedb") { // monkey patch inspect() values to avoid useless NeDB // struct spam on the debug API. this._bridge.getUserStore().inspect = function(depth) { @@ -334,7 +340,6 @@ IrcBridge.prototype.run = Promise.coroutine(function*(port) { } let pkeyPath = this.config.ircService.passwordEncryptionKeyPath; - const dbConfig = this.config.database; if (dbConfig.engine === "postgres") { log.info("Using PgDataStore for Datastore"); this._dataStore = new PgDataStore(this.config.homeserver.domain, dbConfig.connectionString, pkeyPath); @@ -1105,6 +1110,7 @@ IrcBridge.prototype._roomUpgradeMigrateEntry = function(entry, newRoomId) { } IrcBridge.prototype._onRoomUpgrade = Promise.coroutine(function*(oldRoomId, newRoomId) { + yield this.getStore().roomUpgradeOnRoomMigrated(oldRoomId, newRoomId); log.info(`Room has been upgraded from ${oldRoomId} to ${newRoomId}, updating ghosts..`); // Get the channels for the room_id const rooms = yield this.getStore().getIrcChannelsForRoomId(newRoomId); diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index de7282214..5dd972b5d 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -142,5 +142,7 @@ export interface DataStore { getMatrixUserByUsername(domain: string, username: string): Promise; + roomUpgradeOnRoomMigrated(oldRoomId: string, newRoomId: string): Promise; + destroy(): Promise; } diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 4afe3f0a4..91b857767 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -598,6 +598,10 @@ export class NeDBDataStore implements DataStore { return matrixUsers[0]; } + public async roomUpgradeOnRoomMigrated(oldRoomId: string, newRoomId: string) { + // this can no-op, because the matrix-appservice-bridge library will take care of it. + } + public async destroy() { // This will no-op } diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 888866364..c5c2040ed 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -505,6 +505,10 @@ export class PgDataStore implements DataStore { return new MatrixUser(res.rows[0].user_id, res.rows[0].data); } + public async roomUpgradeOnRoomMigrated(oldRoomId: string, newRoomId: string) { + await this.pgPool.query("UPDATE rooms SET room_id = $1 WHERE room_id = $2", [newRoomId, oldRoomId]); + } + public async ensureSchema() { log.info("Starting postgres database engine"); let currentVersion = await this.getSchemaVersion(); From 1fea28317b26532c53eeb5887b4393c8f7402289 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 12:48:03 +0100 Subject: [PATCH 066/350] Features || {} --- src/datastore/postgres/PgDataStore.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index c5c2040ed..34dea2d05 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -26,7 +26,6 @@ import { IrcServer, IrcServerConfig } from "../../irc/IrcServer"; import * as logging from "../../logging"; import Bluebird from "bluebird"; -import { stat } from "fs"; import { StringCrypto } from "../StringCrypto"; import { toIrcLowerCase } from "../../irc/formatting"; @@ -459,7 +458,7 @@ export class PgDataStore implements DataStore { if (pgRes.rowCount === 0) { return {}; } - return pgRes.rows[0].features; + return pgRes.rows[0].features || {}; } public async storeUserFeatures(userId: string, features: UserFeatures): Promise { From 982f3c0e43bfafd78b9255c8ccf07e0f093788a5 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 12:48:15 +0100 Subject: [PATCH 067/350] Use branch for feature for now --- package-lock.json | 11 +++++------ package.json | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 95d552800..0687bd10d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2324,9 +2324,8 @@ } }, "matrix-appservice-bridge": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/matrix-appservice-bridge/-/matrix-appservice-bridge-1.10.3.tgz", - "integrity": "sha512-PqAExCsokZOAnY/d2uqTmB7sCVNKDSHZ9R1V8hDNr6rtMUmO3jIuuKNuwFudeOGL2iYcqzB+s8tCz7yh9a3dew==", + "version": "github:matrix-org/matrix-appservice-bridge#4618830c35698e688c1f4ca8459160e5965e9d1e", + "from": "github:matrix-org/matrix-appservice-bridge#hs/option-to-disable-stores", "requires": { "bluebird": "^2.9.34", "chalk": "^2.4.1", @@ -2417,9 +2416,9 @@ } }, "matrix-js-sdk": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/matrix-js-sdk/-/matrix-js-sdk-2.3.0.tgz", - "integrity": "sha512-jeswie7cWK7+XxcD+pQ7LplWnWkOQDa+x6y7FUUnxCdEvaj38cE5Obo9bPMjFgOln2hISlLdR8fzMNE9F4oUJA==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/matrix-js-sdk/-/matrix-js-sdk-2.4.1.tgz", + "integrity": "sha512-5mOp396eOtvaMiuUD85TWvuxSP532PuvtH/QLugBGenI15FGwtnC40cTnVYviYWGBi340FPrOKWulc5ILRX6qQ==", "requires": { "another-json": "^0.2.0", "babel-runtime": "^6.26.0", diff --git a/package.json b/package.json index 9441b498e..5cc2b7046 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "iconv": "^2.3.4", "irc": "matrix-org/node-irc#matrix-irc-bridge", "js-yaml": "^3.2.7", - "matrix-appservice-bridge": "^1.10.3", + "matrix-appservice-bridge": "matrix-org/matrix-appservice-bridge#hs/option-to-disable-stores", "matrix-lastactive": "^0.0.8", "nedb": "^1.1.2", "nopt": "^3.0.1", From 278c5aa47c17eceb1c5829d033d31b30b46f5e74 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 13:13:26 +0100 Subject: [PATCH 068/350] Fix linting --- src/datastore/NedbDataStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 91b857767..c6b705d59 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -598,7 +598,7 @@ export class NeDBDataStore implements DataStore { return matrixUsers[0]; } - public async roomUpgradeOnRoomMigrated(oldRoomId: string, newRoomId: string) { + public async roomUpgradeOnRoomMigrated() { // this can no-op, because the matrix-appservice-bridge library will take care of it. } From d51eb3eda9512f272c7347e596392fc89d6fe15f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 13:13:59 +0100 Subject: [PATCH 069/350] Newsfile --- changelog.d/824.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/824.feature diff --git a/changelog.d/824.feature b/changelog.d/824.feature new file mode 100644 index 000000000..f864afa63 --- /dev/null +++ b/changelog.d/824.feature @@ -0,0 +1 @@ +Support room upgrades on PostgreSQL. \ No newline at end of file From 64ae05fd35ed30958568ea27f31f56ecb71c8e5f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 1 Oct 2019 10:06:43 +0100 Subject: [PATCH 070/350] Never allow "use strict" --- .ts.eslintrc | 3 ++- src/models/IrcUser.ts | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.ts.eslintrc b/.ts.eslintrc index d0c9fbcae..e2f5b052e 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -5,6 +5,7 @@ "rules": { "@typescript-eslint/ban-ts-ignore": 0, "@typescript-eslint/explicit-function-return-type": 0, - "@typescript-eslint/camelcase": ["error", { "properties": "never" }] + "@typescript-eslint/camelcase": ["error", { "properties": "never" }], + "strict": ["error", "never" ], } } diff --git a/src/models/IrcUser.ts b/src/models/IrcUser.ts index df73ac015..d5ca714a0 100644 --- a/src/models/IrcUser.ts +++ b/src/models/IrcUser.ts @@ -1,4 +1,18 @@ -"use strict"; +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ import { RemoteUser } from "matrix-appservice-bridge"; import { IrcServer } from "../irc/IrcServer"; From 3a66926a0f04230b98af1ca5f2ba5b51e0cd08a6 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 14:25:46 +0100 Subject: [PATCH 071/350] Rewrite ident.js in Typescript --- src/irc/BridgedClient.js | 16 ++-- src/irc/Ident.ts | 160 +++++++++++++++++++++++++++++++++++++++ src/irc/ident.js | 124 ------------------------------ src/main.js | 2 +- 4 files changed, 170 insertions(+), 132 deletions(-) create mode 100644 src/irc/Ident.ts delete mode 100644 src/irc/ident.js diff --git a/src/irc/BridgedClient.js b/src/irc/BridgedClient.js index da55cad51..da5bd632c 100644 --- a/src/irc/BridgedClient.js +++ b/src/irc/BridgedClient.js @@ -5,7 +5,7 @@ const Promise = require("bluebird"); const promiseutil = require("../promiseutil"); const util = require("util"); const EventEmitter = require("events").EventEmitter; -const ident = require("./ident"); +const ident = require("./Ident"); const ConnectionInstance = require("./ConnectionInstance"); const { IrcRoom } = require("../models/IrcRoom"); const log = require("../logging").get("BridgedClient"); @@ -127,7 +127,7 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { `Connecting to the IRC network '${this.server.domain}' as ${this.nick}...` ); - let identToken = ident.takeToken(); + let identResolver = ident.clientBegin(); let connInst = yield ConnectionInstance.create(server, { nick: this.nick, @@ -140,7 +140,7 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { this.server.getIpv6Prefix() ? this._clientConfig.getIpv6Address() : undefined ) }, (inst) => { - this._onConnectionCreated(inst, nameInfo, identToken); + this._onConnectionCreated(inst, nameInfo, identResolver); }); this.inst = connInst; @@ -192,7 +192,9 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { catch (err) { this.log.debug("Failed to connect."); this.instCreationFailed = true; - ident.returnToken(identToken); + if (identResolver) { + identResolver(); + } throw err; } }); @@ -585,7 +587,7 @@ BridgedClient.prototype._addChannel = function(channel) { BridgedClient.prototype.getLastActionTs = function() { return this.lastActionTs; }; -BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, identToken) { +BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, identResolver) { // listen for a connect event which is done when the TCP connection is // established and set ident info (this is different to the connect() callback // in node-irc which actually fires on a registered event..) @@ -597,7 +599,7 @@ BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, iden if (localPort > 0) { ident.setMapping(nameInfo.username, localPort); } - ident.returnToken(identToken); + identResolver(); }); connInst.onDisconnect = (reason) => { @@ -612,7 +614,7 @@ BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, iden "' has been lost. " ); clearTimeout(this._idleTimeout); - ident.returnToken(identToken); + identResolver(); }; this._eventBroker.addHooks(this, connInst); diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts new file mode 100644 index 000000000..4d07c0768 --- /dev/null +++ b/src/irc/Ident.ts @@ -0,0 +1,160 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import net from "net"; +import { promisify } from "util"; + +const log = require("../logging").get("irc-ident"); + + + +interface IdentConfig { + port: number; + address: string; +} + +const DEFAULT_CONFIG = { + port: 113, + address: "0.0.0.0" +}; + +const CLIENT_CONNECTION_TIMEOUT_MS = 120000; + +/** + * Runs an ident server to auth a list of usernames. + * + * This purposefully has no dependencies on any other library and is kept as + * generic as possible. It consists of three functions: + * + * configure(opts) : opts => { port: {Number} } + * Configure the ident server. + * + * run() + * Start listening on the configured port for incoming requests. + * + * setMapping(username, port) : username => {String}, port => {Number} + * Assign a username/port mapping. Setting a port of 0 removes the mapping. + **/ +class IdentSrv { + private config: IdentConfig = DEFAULT_CONFIG; + private portMappings: {[port: string]: string} = {}; + private pendingConnections: Set> = new Set(); + + constructor() { + + } + + public run() { + net.createServer( + this.onConnection.bind(this) + ).listen(this.config.port, this.config.address); + } + + public configure(opts: IdentConfig) { + log.info("Configuring ident server => %s", JSON.stringify(opts)); + this.config = opts; + } + + public setMapping(username: string, port: number) { + if (port > 0) { + this.portMappings[port] = username; + log.debug("Set user %s on port %s", username, port); + } + else if (port === 0) { + Object.keys(this.portMappings) + .filter((portNum: string) => this.portMappings[portNum] === username) + .forEach((portNum) => { + if (this.portMappings[portNum] === username) { + delete this.portMappings[portNum]; + log.debug("Remove user %s from port %s", username, portNum); + } + }); + } + } + + private onConnection(sock: net.Socket) { + log.debug("CONNECT %s %s", sock.remoteAddress, sock.remotePort); + sock.on("data", (data) => { + log.debug("DATA " + data); + const ports = data.toString().split(","); + const remoteConnectPort = Number(ports[1]); + const localOutgoingPort = Number(ports[0]); + if (!remoteConnectPort || !localOutgoingPort) { + log.debug("BAD DATA"); + sock.end(); + return; + } + this.tryRespond(sock, + String(localOutgoingPort), + String(remoteConnectPort)).catch((ex) => { + // Just close the connection + sock.end(); + }); + }); + sock.on("close", () => { + log.debug("CLOSE"); + }); + sock.on("error", (err) => { + log.error("connection error: " + err); + if (err && err.stack) { + log.error(err.stack); + } + }); + } + + public clientBegin(): () => void { + log.debug("IRC client started connection"); + let res!: () => void; + const p: Promise = new Promise((resolve) => { + res = resolve; + setTimeout(resolve, CLIENT_CONNECTION_TIMEOUT_MS); + }); + this.pendingConnections.add(p); + p.then(() => { + log.debug("IRC client connected"); + this.pendingConnections.delete(p); + }) + return res; + } + + private async tryRespond(sock: net.Socket, localPort: string, remotePort: string) { + const username = this.portMappings[localPort]; + if (username) { + log.debug("Port %s is %s", localPort, username); + this.respond(sock, localPort, remotePort, username); + return; + } + // Wait for pending connections to finish first. + await Promise.all([...this.pendingConnections]); + log.debug("No user on port %s", localPort); + this.respond(sock, localPort, remotePort); + } + + private respond(sock: net.Socket, localPort: string, remotePort: string, username?: string) { + let response; + if (username) { + response = `${localPort},${remotePort}:USERID:UNIX:${username}\r\n`; + } + else { + response = `${localPort},${remotePort}:ERROR:NO-USER\r\n`; + } + log.debug(response); + sock.end(response); + } +} + +const staticInstance = new IdentSrv(); +module.exports = staticInstance; \ No newline at end of file diff --git a/src/irc/ident.js b/src/irc/ident.js deleted file mode 100644 index d25b42c45..000000000 --- a/src/irc/ident.js +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Runs an ident server to auth a list of usernames. - * - * This purposefully has no dependencies on any other library and is kept as - * generic as possible. It consists of three functions: - * - * configure(opts) : opts => { port: {Number} } - * Configure the ident server. - * - * run() - * Start listening on the configured port for incoming requests. - * - * setMapping(username, port) : username => {String}, port => {Number} - * Assign a username/port mapping. Setting a port of 0 removes the mapping. - */ -"use strict"; - -const EventEmitter = require('events'); -const net = require('net'); - -const log = require("../logging").get("irc-ident"); - -var config = { - port: 113, - address: "0.0.0.0" -}; -var portMappings = { - // port: username -}; - -var nextToken = 1; -const openTokens = []; -const emitter = new EventEmitter(); - -var respond = function(sock, localPort, remotePort, username) { - var response; - if (username) { - response = localPort + "," + remotePort + ":USERID:UNIX:" + username; - } - else { - response = localPort + "," + remotePort + ":ERROR:NO-USER"; - } - response += "\r\n"; - - log.debug(response); - sock.end(response); -}; - -const tryRespond = (currentToken, sock, localPort, remotePort) => { - let username = portMappings[localPort]; - if (username) { - log.debug("Port %s is %s", localPort, username); - respond(sock, localPort, remotePort, username); - } - else if (openTokens.length > 0 && openTokens[0] < currentToken) { - emitter.once("token", () => tryRespond(currentToken, sock, localPort, remotePort)); - } - else { - log.debug("No user on port %s", localPort); - respond(sock, localPort, remotePort, null); - } -}; - -module.exports = { - configure: function(opts) { - log.info("Configuring ident server => %s", JSON.stringify(opts)); - config = opts; - }, - run: function() { - net.createServer(function(sock) { - log.debug("CONNECT %s %s", sock.remoteAddress, sock.remotePort); - sock.on("data", function(data) { - log.debug("DATA " + data); - var ports = data.toString().split(","); - var remoteConnectPort = Number(ports[1]); - var localOutgoingPort = Number(ports[0]); - if (!remoteConnectPort || !localOutgoingPort) { - log.debug("BAD DATA"); - return; - } - tryRespond(nextToken, sock, - String(localOutgoingPort), - String(remoteConnectPort)); - }); - sock.on("close", function() { - log.debug("CLOSE"); - }); - sock.on("error", function(err) { - log.error("connection error: " + err); - if (err && err.stack) { - log.error(err.stack); - } - }); - }).listen(config.port, config.address); - }, - setMapping: function(username, port) { - if (port) { - portMappings[port] = username; - log.debug("Set user %s on port %s", username, port); - } - else if (port === 0) { - Object.keys(portMappings).forEach(function(portNum) { - if (portMappings[portNum] === username) { - portMappings[portNum] = undefined; - log.debug("Remove user %s from port %s", username, portNum); - } - }); - } - }, - takeToken: function() { - let token = nextToken++; - openTokens.push(token); - log.debug("Took token %d", token); - return token; - }, - returnToken: function(token) { - let index = openTokens.indexOf(token); - if (index > -1) { - openTokens.splice(index, 1); - log.debug("Returned token %d", token); - emitter.emit('token'); - } - } -}; diff --git a/src/main.js b/src/main.js index 40ec5a339..4d68b32d1 100644 --- a/src/main.js +++ b/src/main.js @@ -9,7 +9,7 @@ const UserBridgeStore = require("matrix-appservice-bridge").UserBridgeStore; const IrcBridge = require("./bridge/IrcBridge.js"); const { IrcServer } = require("./irc/IrcServer.js"); const stats = require("./config/stats"); -const ident = require("./irc/ident"); +const ident = require("./irc/Ident"); const logging = require("./logging"); const log = logging.get("main"); From 22e4987b6d9b68d670a8cc0ffe8f71cfa2ac3d6a Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 14:38:38 +0100 Subject: [PATCH 072/350] Linting --- src/irc/Ident.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 4d07c0768..1b174d8e6 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -15,12 +15,9 @@ limitations under the License. */ import net from "net"; -import { promisify } from "util"; const log = require("../logging").get("irc-ident"); - - interface IdentConfig { port: number; address: string; @@ -53,10 +50,6 @@ class IdentSrv { private portMappings: {[port: string]: string} = {}; private pendingConnections: Set> = new Set(); - constructor() { - - } - public run() { net.createServer( this.onConnection.bind(this) @@ -99,7 +92,7 @@ class IdentSrv { } this.tryRespond(sock, String(localOutgoingPort), - String(remoteConnectPort)).catch((ex) => { + String(remoteConnectPort)).catch(() => { // Just close the connection sock.end(); }); @@ -115,7 +108,7 @@ class IdentSrv { }); } - public clientBegin(): () => void { + public clientBegin(): () => void { log.debug("IRC client started connection"); let res!: () => void; const p: Promise = new Promise((resolve) => { @@ -146,15 +139,15 @@ class IdentSrv { private respond(sock: net.Socket, localPort: string, remotePort: string, username?: string) { let response; if (username) { - response = `${localPort},${remotePort}:USERID:UNIX:${username}\r\n`; + response = `${localPort}, ${remotePort}:USERID:UNIX:${username}\r\n`; } else { - response = `${localPort},${remotePort}:ERROR:NO-USER\r\n`; - } + response = `${localPort}, ${remotePort}:ERROR:NO-USER\r\n`; + } log.debug(response); sock.end(response); } } const staticInstance = new IdentSrv(); -module.exports = staticInstance; \ No newline at end of file +module.exports = staticInstance; From 7db797dba6bb921fa7de08d86cc91c824c73490d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 14:39:28 +0100 Subject: [PATCH 073/350] Add newsfile --- changelog.d/825.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/825.feature diff --git a/changelog.d/825.feature b/changelog.d/825.feature new file mode 100644 index 000000000..1dc6b249c --- /dev/null +++ b/changelog.d/825.feature @@ -0,0 +1 @@ +Delay ident responses until pending clients have connected. Thanks to @heftig for the initial PR. \ No newline at end of file From 0dac9b4c46e64588be40817598df54704ad10051 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:04:32 +0100 Subject: [PATCH 074/350] Retroactively commit changelog entries for older PRs --- changelog.d/808.feature | 1 + changelog.d/809.misc | 1 + changelog.d/810.misc | 1 + changelog.d/812.misc | 1 + changelog.d/814.misc | 1 + changelog.d/816.feature | 1 + changelog.d/819.misc | 1 + 7 files changed, 7 insertions(+) create mode 100644 changelog.d/808.feature create mode 100644 changelog.d/809.misc create mode 100644 changelog.d/810.misc create mode 100644 changelog.d/812.misc create mode 100644 changelog.d/814.misc create mode 100644 changelog.d/816.feature create mode 100644 changelog.d/819.misc diff --git a/changelog.d/808.feature b/changelog.d/808.feature new file mode 100644 index 000000000..192c9dd27 --- /dev/null +++ b/changelog.d/808.feature @@ -0,0 +1 @@ +The project now uses Typescript for it's source code. \ No newline at end of file diff --git a/changelog.d/809.misc b/changelog.d/809.misc new file mode 100644 index 000000000..21716e53b --- /dev/null +++ b/changelog.d/809.misc @@ -0,0 +1 @@ +Refactor Datastore for Typescript \ No newline at end of file diff --git a/changelog.d/810.misc b/changelog.d/810.misc new file mode 100644 index 000000000..509b97349 --- /dev/null +++ b/changelog.d/810.misc @@ -0,0 +1 @@ +Add linting support for Typescript files. \ No newline at end of file diff --git a/changelog.d/812.misc b/changelog.d/812.misc new file mode 100644 index 000000000..9132e83ef --- /dev/null +++ b/changelog.d/812.misc @@ -0,0 +1 @@ +Fatal exceptions are now logged to stdout in addition to logs. \ No newline at end of file diff --git a/changelog.d/814.misc b/changelog.d/814.misc new file mode 100644 index 000000000..0a468e4b3 --- /dev/null +++ b/changelog.d/814.misc @@ -0,0 +1 @@ +Refactor Datastore code to be more generic. \ No newline at end of file diff --git a/changelog.d/816.feature b/changelog.d/816.feature new file mode 100644 index 000000000..d5c6fc901 --- /dev/null +++ b/changelog.d/816.feature @@ -0,0 +1 @@ +Add migration script for migrating NeDB databases to PostgreSQL. \ No newline at end of file diff --git a/changelog.d/819.misc b/changelog.d/819.misc new file mode 100644 index 000000000..03ef47784 --- /dev/null +++ b/changelog.d/819.misc @@ -0,0 +1 @@ +Move schema.yml from /lib/config to / \ No newline at end of file From e2c4057b52fe3d5f6df533cbbac78c782f786d8c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:08:43 +0100 Subject: [PATCH 075/350] Remove defunct schema.yml file --- src/config/schema.yml | 323 ------------------------------------------ 1 file changed, 323 deletions(-) delete mode 100644 src/config/schema.yml diff --git a/src/config/schema.yml b/src/config/schema.yml deleted file mode 100644 index 8a102cd78..000000000 --- a/src/config/schema.yml +++ /dev/null @@ -1,323 +0,0 @@ -"$schema": "http://json-schema.org/draft-04/schema#" -type: "object" -properties: - advanced: - type: "object" - properties: - maxHttpSockets: - type: "integer" - homeserver: - type: "object" - properties: - url: - type: "string" - media_url: - type: "string" - domain: - type: "string" - dropMatrixMessagesAfterSecs: - type: "integer" - enablePresence: - type: "boolean" - required: ["url", "domain"] - ircService: - type: "object" - properties: - databaseUri: - type: "string" - metrics: - type: "object" - properties: - enabled: - type: "boolean" - remoteUserAgeBuckets: - type: "array" - items: - type: "string" - pattern: "^[0-9]+(h|d|w)$" - statsd: - type: "object" - properties: - hostname: - type: "string" - port: - type: "integer" - jobName: - type: "string" - required: ["hostname", "port"] - ident: - type: "object" - properties: - enabled: - type: "boolean" - port: - type: "integer" - address: - type: "string" - required: ["enabled"] - debugApi: - type: "object" - properties: - enabled: - type: "boolean" - port: - type: "integer" - required: ["enabled", "port"] - logging: - type: "object" - properties: - level: - type: "string" - enum: ["error","warn","info","debug"] - logfile: - type: "string" - errfile: - type: "string" - toConsole: - type: "boolean" - maxFileSizeBytes: - type: "integer" - maxFiles: - type: "integer" - provisioning: - type: "object" - properties: - enabled: - type: "boolean" - requestTimeoutSeconds: - type: "number" - ruleFile: - type: "string" - enableReload: - type: "boolean" - passwordEncryptionKeyPath: - type: "string" - matrixHandler: - type: "object" - properties: - eventCacheSize: - type: "integer" - ircHandler: - type: "object" - properties: - leaveConcurrency: - type: "integer" - mapIrcMentionsToMatrix: - type: "string" - enum: ["on", "off", "force-off"] - servers: - type: "object" - # all properties must follow the following - additionalProperties: - type: "object" - properties: - port: - type: "integer" - additionalAddresses: - type: "array" - items: - type: "string" - ssl: - type: "boolean" - sslselfsign: - type: "boolean" - sasl: - type: "boolean" - allowExpiredCerts: - type: "boolean" - password: - type: "string" - sendConnectionMessages: - type: "boolean" - name: - type: "string" - description: - type: "string" - networkId: - type: "string" - pattern: "^[a-zA-Z0-9]+$" - icon: - type: "string" - quitDebounce: - type: "object" - properties: - enabled: - type: "boolean" - quitsPerSecond: - type: "number" - delayMinMs: - type: "integer" - minimum: 0 - exclusiveMinimum: true - delayMaxMs: - type: "integer" - minimum: 0 - exclusiveMinimum: true - modePowerMap: - type: "object" - patternProperties: - # Single character modes mapped to positive power levels - "^[a-zA-Z]$": - type: number - minimum: 0 - botConfig: - type: "object" - properties: - enabled: - type: "boolean" - nick: - type: "string" - password: - type: "string" - joinChannelsIfNoUsers: - type: "boolean" - privateMessages: - type: "object" - properties: - enabled: - type: "boolean" - exclude: - type: "array" - items: - type: "string" - federate: - type: "boolean" - membershipLists: - type: "object" - properties: - enabled: - type: "boolean" - floodDelayMs: - type: "integer" - global: - type: "object" - properties: - ircToMatrix: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - matrixToIrc: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - rooms: - type: "array" - items: - type: "object" - properties: - room: - type: "string" - pattern: "^!+.*$" - matrixToIrc: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - channels: - type: "array" - items: - type: "object" - properties: - channel: - type: "string" - pattern: "^#+.*$" - ircToMatrix: - type: "object" - properties: - initial: - type: "boolean" - incremental: - type: "boolean" - additionalProperties: false - dynamicChannels: - type: "object" - properties: - enabled: - type: "boolean" - published: - type: "boolean" - createAlias: - type: "boolean" - groupId: - type: "string" - joinRule: - type: "string" - enum: ["invite", "public"] - federate: - type: "boolean" - roomVersion: - type: "string" - aliasTemplate: - type: "string" - pattern: "^#.*\\$CHANNEL" - whitelist: - type: "array" - items: - type: "string" - pattern: "^@.*" - exclude: - type: "array" - items: - type: "string" - mappings: - type: "object" - patternProperties: - # must start with a # - "^#+.*$": - type: "array" - items: - type: "string" - minItems: 1 - uniqueItems: true - additionalProperties: false - matrixClients: - type: "object" - properties: - userTemplate: - type: "string" - pattern: "^@.*\\$NICK" - displayName: - type: "string" - pattern: "\\$NICK" - joinAttempts: - type: "integer" - minimum: -1 - ircClients: - type: "object" - properties: - nickTemplate: - type: "string" - pattern: "\\$USERID|\\$LOCALPART|\\$DISPLAY" - maxClients: - type: "integer" - idleTimeout: - type: "integer" - minimum: 0 - reconnectIntervalMs: - type: "integer" - minimum: 0 - allowNickChanges: - type: "boolean" - ipv6: - type: "object" - properties: - prefix: - type: "string" - pattern: "[ABCDEFabcdef0123456789:]+" - only: - type: "boolean" - lineLimit: - type: "integer" - userModes: - type: "string" - required: ["databaseUri", "servers"] From 4a0629cc9329b0f39fdc7e901bdb97afe504bd87 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:12:45 +0100 Subject: [PATCH 076/350] Add newsfile --- changelog.d/820.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/820.feature diff --git a/changelog.d/820.feature b/changelog.d/820.feature new file mode 100644 index 000000000..448ff8399 --- /dev/null +++ b/changelog.d/820.feature @@ -0,0 +1 @@ +Add config option `excludedUsers` to exclude users from bridging by regex. \ No newline at end of file From ff0f2a2e5d3d17cfc71743e7fcdfcd15affb7630 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:35:28 +0100 Subject: [PATCH 077/350] Remove spaces --- src/irc/Ident.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 1b174d8e6..129ac1d58 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -139,10 +139,10 @@ class IdentSrv { private respond(sock: net.Socket, localPort: string, remotePort: string, username?: string) { let response; if (username) { - response = `${localPort}, ${remotePort}:USERID:UNIX:${username}\r\n`; + response = `${localPort},${remotePort}:USERID:UNIX:${username}\r\n`; } else { - response = `${localPort}, ${remotePort}:ERROR:NO-USER\r\n`; + response = `${localPort},${remotePort}:ERROR:NO-USER\r\n`; } log.debug(response); sock.end(response); From b084295fca0d98eca217624bcea1b393b36ebcd7 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:35:35 +0100 Subject: [PATCH 078/350] Compare hashes to determine if the portMappings have changed. --- src/irc/Ident.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 129ac1d58..2af4a3370 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -123,15 +123,21 @@ class IdentSrv { return res; } - private async tryRespond(sock: net.Socket, localPort: string, remotePort: string) { + private async tryRespond(sock: net.Socket, localPort: string, remotePort: string): Promise { const username = this.portMappings[localPort]; if (username) { log.debug("Port %s is %s", localPort, username); this.respond(sock, localPort, remotePort, username); return; } + const mappingHash = Object.keys(this.portMappings).join("|"); // Wait for pending connections to finish first. await Promise.all([...this.pendingConnections]); + // We don't know here if anything has actually changed, so compare hashes. + if (Object.keys(this.portMappings).join("|") !== mappingHash) { + // Hash has changed, retry. + return this.tryRespond(sock, localPort, remotePort); + } log.debug("No user on port %s", localPort); this.respond(sock, localPort, remotePort); } From 412352ad0b5b215bbcffe50caca4f0ff727f8635 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 15:38:39 +0100 Subject: [PATCH 079/350] Remove Promise --- src/irc/Ident.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 2af4a3370..0c381491f 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -123,7 +123,7 @@ class IdentSrv { return res; } - private async tryRespond(sock: net.Socket, localPort: string, remotePort: string): Promise { + private async tryRespond(sock: net.Socket, localPort: string, remotePort: string) { const username = this.portMappings[localPort]; if (username) { log.debug("Port %s is %s", localPort, username); @@ -136,7 +136,8 @@ class IdentSrv { // We don't know here if anything has actually changed, so compare hashes. if (Object.keys(this.portMappings).join("|") !== mappingHash) { // Hash has changed, retry. - return this.tryRespond(sock, localPort, remotePort); + await this.tryRespond(sock, localPort, remotePort); + return; } log.debug("No user on port %s", localPort); this.respond(sock, localPort, remotePort); From 6c54ecc8ca9063b0121692c15c84b393c3e4bd7a Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 16:06:45 +0100 Subject: [PATCH 080/350] Incorporate feedback, merge tryRespond with respond --- src/irc/Ident.ts | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 0c381491f..a4435aec3 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -90,7 +90,7 @@ class IdentSrv { sock.end(); return; } - this.tryRespond(sock, + this.respond(sock, String(localOutgoingPort), String(remoteConnectPort)).catch(() => { // Just close the connection @@ -123,32 +123,20 @@ class IdentSrv { return res; } - private async tryRespond(sock: net.Socket, localPort: string, remotePort: string) { - const username = this.portMappings[localPort]; - if (username) { - log.debug("Port %s is %s", localPort, username); - this.respond(sock, localPort, remotePort, username); - return; - } - const mappingHash = Object.keys(this.portMappings).join("|"); - // Wait for pending connections to finish first. - await Promise.all([...this.pendingConnections]); - // We don't know here if anything has actually changed, so compare hashes. - if (Object.keys(this.portMappings).join("|") !== mappingHash) { - // Hash has changed, retry. - await this.tryRespond(sock, localPort, remotePort); - return; + private async respond(sock: net.Socket, localPort: string, remotePort: string) { + let username = this.portMappings[localPort]; + if (!username) { + // Wait for pending connections to finish first. + await Promise.all([...this.pendingConnections]); + username = this.portMappings[localPort]; } - log.debug("No user on port %s", localPort); - this.respond(sock, localPort, remotePort); - } - private respond(sock: net.Socket, localPort: string, remotePort: string, username?: string) { let response; if (username) { + log.debug("Port %s is %s", localPort, username); response = `${localPort},${remotePort}:USERID:UNIX:${username}\r\n`; - } - else { + } else { + log.debug("No user on port %s", localPort); response = `${localPort},${remotePort}:ERROR:NO-USER\r\n`; } log.debug(response); From 570392ec01f226be502a1813255318fb3f24ba17 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 16:10:38 +0100 Subject: [PATCH 081/350] Convert promiseutil to Typescript --- src/promiseutil.js | 23 ----------------------- src/promiseutil.ts | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 23 deletions(-) delete mode 100644 src/promiseutil.js create mode 100644 src/promiseutil.ts diff --git a/src/promiseutil.js b/src/promiseutil.js deleted file mode 100644 index b0ec46aed..000000000 --- a/src/promiseutil.js +++ /dev/null @@ -1,23 +0,0 @@ -const Promise = require("bluebird"); - -function defer() { - var resolve, reject; - var promise = new Promise(function() { - resolve = arguments[0]; - reject = arguments[1]; - }); - return { - resolve: resolve, - reject: reject, - promise: promise - }; -} - -function allSettled(promises) { - return Promise.all(promises.map(function(p) { - return p.reflect(); - })); -} - -module.exports.defer = defer; -module.exports.allSettled = allSettled; diff --git a/src/promiseutil.ts b/src/promiseutil.ts new file mode 100644 index 000000000..fc9f4fdcd --- /dev/null +++ b/src/promiseutil.ts @@ -0,0 +1,36 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import Bluebird from "bluebird"; + +export function defer() { + let resolve, reject; + const promise = new Bluebird(function() { + resolve = arguments[0]; + reject = arguments[1]; + }); + return { + resolve: resolve, + reject: reject, + promise: promise + }; +} + +export function allSettled(promises: Bluebird[]) { + return Bluebird.all(promises.map(function(p) { + return p.reflect(); + })); +} From 3d1a2cbca568e124ae62115846fc5ca2278c812f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 16:53:18 +0100 Subject: [PATCH 082/350] Port Queue to Typescript --- spec/unit/Queue.spec.js | 2 +- src/bridge/IrcHandler.js | 2 +- src/bridge/MemberListSyncer.js | 2 +- src/irc/IdentGenerator.js | 2 +- src/irc/Ipv6Generator.js | 2 +- src/irc/Scheduler.js | 2 +- src/promiseutil.ts | 21 +++-- src/util/Queue.js | 134 --------------------------- src/util/Queue.ts | 160 +++++++++++++++++++++++++++++++++ 9 files changed, 180 insertions(+), 147 deletions(-) delete mode 100644 src/util/Queue.js create mode 100644 src/util/Queue.ts diff --git a/spec/unit/Queue.spec.js b/spec/unit/Queue.spec.js index a8d16ef6c..3beb71bd9 100644 --- a/spec/unit/Queue.spec.js +++ b/spec/unit/Queue.spec.js @@ -1,6 +1,6 @@ "use strict"; const Promise = require("bluebird"); -const Queue = require("../../lib/util/Queue.js"); +const { Queue } = require("../../lib/util/Queue.js"); const test = require("../util/test"); describe("Queue", function() { diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index ceb046cc6..0010429ba 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -7,7 +7,7 @@ const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const MatrixAction = require("../models/MatrixAction"); -const Queue = require("../util/Queue.js"); +const { Queue } = require("../util/Queue.js"); const QueuePool = require("../util/QueuePool.js"); const QuitDebouncer = require("./QuitDebouncer.js"); const RoomAccessSyncer = require("./RoomAccessSyncer.js"); diff --git a/src/bridge/MemberListSyncer.js b/src/bridge/MemberListSyncer.js index c0910ce8b..9a3ff55d9 100644 --- a/src/bridge/MemberListSyncer.js +++ b/src/bridge/MemberListSyncer.js @@ -9,7 +9,7 @@ const promiseutil = require("../promiseutil"); const log = require("../logging").get("MemberListSyncer"); const stats = require("../config/stats"); const QueuePool = require("../util/QueuePool"); -const Queue = require("../util/Queue"); +const { Queue } = require("../util/Queue"); function MemberListSyncer(ircBridge, appServiceBot, server, appServiceUserId, injectJoinFn) { this.ircBridge = ircBridge; diff --git a/src/irc/IdentGenerator.js b/src/irc/IdentGenerator.js index 6fecd7ab8..8c06bd402 100644 --- a/src/irc/IdentGenerator.js +++ b/src/irc/IdentGenerator.js @@ -1,7 +1,7 @@ /*eslint no-invalid-this: 0 no-constant-condition: 0 */ "use strict"; const Promise = require("bluebird"); -const Queue = require("../util/Queue"); +const { Queue } = require("../util/Queue"); const log = require("../logging").get("IdentGenerator"); function IdentGenerator(store) { diff --git a/src/irc/Ipv6Generator.js b/src/irc/Ipv6Generator.js index e39292091..c86b96014 100644 --- a/src/irc/Ipv6Generator.js +++ b/src/irc/Ipv6Generator.js @@ -1,7 +1,7 @@ /*eslint no-invalid-this: 0 */ "use strict"; const Promise = require("bluebird"); -const Queue = require("../util/Queue"); +const { Queue } = require("../util/Queue"); const log = require("../logging").get("Ipv6Generator"); function Ipv6Generator(store) { diff --git a/src/irc/Scheduler.js b/src/irc/Scheduler.js index 7445fed42..40676130c 100644 --- a/src/irc/Scheduler.js +++ b/src/irc/Scheduler.js @@ -3,7 +3,7 @@ const Promise = require("bluebird"); const logging = require("../logging"); var log = logging.get("scheduler"); -const Queue = require("../util/Queue.js"); +const { Queue } = require("../util/Queue.js"); /** * An IRC connection scheduler. Enables ConnectionInstance to reconnect diff --git a/src/promiseutil.ts b/src/promiseutil.ts index fc9f4fdcd..be2fc0c96 100644 --- a/src/promiseutil.ts +++ b/src/promiseutil.ts @@ -16,15 +16,22 @@ limitations under the License. import Bluebird from "bluebird"; -export function defer() { - let resolve, reject; - const promise = new Bluebird(function() { - resolve = arguments[0]; - reject = arguments[1]; +export interface Defer { + resolve: (value?: T) => void; + reject: (err?: unknown) => void; + promise: Promise; +} + +export function defer(): Defer { + let resolve: (value?: T) => void; + let reject: (err?: unknown) => void; + const promise = new Bluebird((res, rej) => { + resolve = res; + reject = rej }); return { - resolve: resolve, - reject: reject, + resolve: resolve!, + reject: reject!, promise: promise }; } diff --git a/src/util/Queue.js b/src/util/Queue.js deleted file mode 100644 index b0cce126d..000000000 --- a/src/util/Queue.js +++ /dev/null @@ -1,134 +0,0 @@ -/*eslint no-invalid-this: 0 */ -"use strict"; -const Promise = require("bluebird"); -const promiseutil = require("../promiseutil"); - -/** - * Construct a new Queue which will process items FIFO. - * @param {Function} processFn The function to invoke when the item being processed - * is in its critical section. Only 1 item at any one time will be calling this function. - * The function should return a Promise which is resolved/rejected when the next item - * can be taken from the queue. - * @param {integer} intervalMs Optional. If provided and > 0, this queue will be serviced - * at an interval of intervalMs. Otherwise, items will be processed as soon as they become - * the first item in the queue to be processed. - */ -function Queue(processFn, intervalMs) { - this._queue = []; - this._processing = null; - this._procFn = processFn; // critical section Promise = fn(item) - this._onceFreeDefers = []; - - if (intervalMs !== undefined && !(Number.isInteger(intervalMs) && intervalMs >= 0) ) { - throw new Error('intervalMs must be a positive integer'); - } - - this._intervalMs = intervalMs; - - if (this._intervalMs) { - // Start consuming - this._consume(); - } -} - -/** - * Return the length of the queue, including the currently processed item. - * @return {Number} The length of the queue. - */ -Queue.prototype.size = function() { - return this._queue.length + (this._processing ? 1 : 0); -}; - -/** - * Return a promise which is resolved when this queue is free (0 items in queue). - * @return {Promise} Resolves to the Queue itself. - */ -Queue.prototype.onceFree = function() { - if (this.size() === 0) { - return Promise.resolve(); - } - let defer = promiseutil.defer(); - this._onceFreeDefers.push(defer); - return defer.promise; -}; - -Queue.prototype._fireOnceFree = function() { - this._onceFreeDefers.forEach((d) => { - d.resolve(this); - }); - this._onceFreeDefers = []; -} - -/** - * Queue up a request for the critical section function. - * @param {string} id An ID to associate with this request. If there is already a - * request with this ID, the promise for that request will be returned. - * @param {*} thing The item to enqueue. It will be passed verbatim to the critical - * section function passed in the constructor. - * @return {Promise} A promise which will be resolved/rejected when the queued item - * has been processed. - */ -Queue.prototype.enqueue = function(id, thing) { - for (var i = 0; i < this._queue.length; i++) { - if (this._queue[i].id === id) { - return this._queue[i].defer.promise; - } - } - let defer = promiseutil.defer(); - this._queue.push({ - id: id, - item: thing, - defer: defer - }); - if (!this._intervalMs) { - // always process stuff asyncly, never syncly. - process.nextTick(() => { - this._consume(); - }); - } - return defer.promise; -}; - -Queue.prototype._retry = function () { - setTimeout(this._consume.bind(this), this._intervalMs); -} - -Queue.prototype._consume = Promise.coroutine(function*() { - if (this._processing) { - return; - } - this._processing = this._queue.shift(); - if (!this._processing) { - if (this._intervalMs) { - this._retry(); - } - this._fireOnceFree(); - return; - } - try { - let thing = this._procFn(this._processing.item); - let result = yield thing; - this._processing.defer.resolve(result); - } - catch (err) { - this._processing.defer.reject(err); - } - finally { - this._processing = null; - if (this._intervalMs) { - this._retry(); - } - } - if (!this._intervalMs) { - this._consume(); - } -}); - -Queue.prototype.killAll = function() { - for (var i = 0; i < this._queue.length; i++) { - this._queue[i].defer.reject(new Error('Queue killed')); - } -} - - -module.exports = Queue; diff --git a/src/util/Queue.ts b/src/util/Queue.ts new file mode 100644 index 000000000..e927a6840 --- /dev/null +++ b/src/util/Queue.ts @@ -0,0 +1,160 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import Bluebird from "bluebird"; +import * as promiseutil from "../promiseutil"; +import { Defer } from "../promiseutil"; + +export interface QueueItem { + id: string; + item: unknown; + defer: Defer; +} + +export type QueueProcessFn = (item: unknown) => Bluebird; + +export class Queue { + private queue: QueueItem[] = []; + private processing: QueueItem|null|undefined = null; + private onceFreeDefers: Defer[] = []; + private consume: () => Bluebird; + + /** + * Construct a new Queue which will process items FIFO. + * @param {Function} processFn The function to invoke when the item being processed + * is in its critical section. Only 1 item at any one time will be calling this function. + * The function should return a Promise which is resolved/rejected when the next item + * can be taken from the queue. + * @param {integer} intervalMs Optional. If provided and > 0, this queue will be serviced + * at an interval of intervalMs. Otherwise, items will be processed as soon as they become + * the first item in the queue to be processed. + */ + constructor(private processFn: QueueProcessFn, private intervalMs?: number) { + if (intervalMs !== undefined && !(Number.isInteger(intervalMs) && intervalMs >= 0) ) { + throw Error('intervalMs must be a positive integer'); + } + + // XXX: Coroutines have subtly different behaviour to async/await functions + // and I've not managed to track down precisely why. For the sake of keeping the + // QueuePool tests happy, we will continue to use coroutine functions for now. + this.consume = Bluebird.coroutine(this.coConsume).bind(this); + + if (intervalMs) { + // Start consuming + this.consume(); + } + + } + + /** + * Return the length of the queue, including the currently processed item. + * @return {Number} The length of the queue. + */ + public size(): number { + return this.queue.length + (this.processing ? 1 : 0); + }; + + /** + * Return a promise which is resolved when this queue is free (0 items in queue). + * @return {Promise} Resolves to the Queue itself. + */ + public onceFree(): Promise { + if (this.size() === 0) { + return Promise.resolve(); + } + const defer = promiseutil.defer(); + this.onceFreeDefers.push(defer); + return defer.promise; + }; + + private fireOnceFree() { + this.onceFreeDefers.forEach((d) => { + d.resolve(this); + }); + this.onceFreeDefers = []; + } + + /** + * Queue up a request for the critical section function. + * @param {string} id An ID to associate with this request. If there is already a + * request with this ID, the promise for that request will be returned. + * @param {*} thing The item to enqueue. It will be passed verbatim to the critical + * section function passed in the constructor. + * @return {Promise} A promise which will be resolved/rejected when the queued item + * has been processed. + */ + public enqueue(id: string, thing: unknown) { + for (let i = 0; i < this.queue.length; i++) { + if (this.queue[i].id === id) { + return this.queue[i].defer.promise; + } + } + const defer = promiseutil.defer(); + this.queue.push({ + id: id, + item: thing, + defer: defer + }); + if (!this.intervalMs) { + // always process stuff asyncly, never syncly. + process.nextTick(() => { + this.consume(); + }); + } + return defer.promise; + }; + + private retry () { + setTimeout(this.consume.bind(this), this.intervalMs); + } + + private* coConsume () { + if (this.processing) { + return; + } + this.processing = this.queue.shift(); + if (!this.processing) { + if (this.intervalMs) { + this.retry(); + } + this.fireOnceFree(); + return; + } + try { + let thing = this.processFn(this.processing.item); + let result = yield thing; + this.processing.defer.resolve(result); + } + catch (err) { + this.processing.defer.reject(err); + } + finally { + this.processing = null; + if (this.intervalMs) { + this.retry(); + } + } + if (!this.intervalMs) { + this.consume(); + } + } + + public killAll() { + for (let i = 0; i < this.queue.length; i++) { + this.queue[i].defer.reject(new Error('Queue killed')); + } + } +} From cc1321bf4eba276ab0189fd536cdcb7f9d1d6e72 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 16:54:18 +0100 Subject: [PATCH 083/350] Port QueuePool to Typescript --- spec/unit/QueuePool.spec.js | 2 +- src/bridge/IrcHandler.js | 4 +- src/bridge/MemberListSyncer.js | 2 +- src/irc/ClientPool.ts | 2 +- src/util/QueuePool.js | 100 --------------------------- src/util/QueuePool.ts | 122 +++++++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 105 deletions(-) delete mode 100644 src/util/QueuePool.js create mode 100644 src/util/QueuePool.ts diff --git a/spec/unit/QueuePool.spec.js b/spec/unit/QueuePool.spec.js index 9a5e38c3f..00857d693 100644 --- a/spec/unit/QueuePool.spec.js +++ b/spec/unit/QueuePool.spec.js @@ -1,5 +1,5 @@ "use strict"; -let QueuePool = require("../../lib/util/QueuePool"); +let { QueuePool } = require("../../lib/util/QueuePool"); let promiseutil = require("../../lib/promiseutil"); let test = require("../util/test"); diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index 0010429ba..ca1c67ea3 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -2,13 +2,13 @@ const Promise = require("bluebird"); const stats = require("../config/stats"); -const { BridgeRequest } = require("../models/BridgeRequest"); +const BridgeRequest = require("../models/BridgeRequest"); const { IrcRoom } = require("../models/IrcRoom"); const MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const MatrixAction = require("../models/MatrixAction"); const { Queue } = require("../util/Queue.js"); -const QueuePool = require("../util/QueuePool.js"); +const { QueuePool } = require("../util/QueuePool.js"); const QuitDebouncer = require("./QuitDebouncer.js"); const RoomAccessSyncer = require("./RoomAccessSyncer.js"); diff --git a/src/bridge/MemberListSyncer.js b/src/bridge/MemberListSyncer.js index 9a3ff55d9..251b832a8 100644 --- a/src/bridge/MemberListSyncer.js +++ b/src/bridge/MemberListSyncer.js @@ -8,7 +8,7 @@ const Promise = require("bluebird"); const promiseutil = require("../promiseutil"); const log = require("../logging").get("MemberListSyncer"); const stats = require("../config/stats"); -const QueuePool = require("../util/QueuePool"); +const { QueuePool } = require("../util/QueuePool"); const { Queue } = require("../util/Queue"); function MemberListSyncer(ircBridge, appServiceBot, server, appServiceUserId, injectJoinFn) { diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 1982e4bdc..239a26993 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -16,7 +16,7 @@ limitations under the License. const stats = require("../config/stats"); const log = require("../logging").get("ClientPool"); -const QueuePool = require("../util/QueuePool"); +const { QueuePool } = require("../util/QueuePool"); import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; diff --git a/src/util/QueuePool.js b/src/util/QueuePool.js deleted file mode 100644 index 1c54735ad..000000000 --- a/src/util/QueuePool.js +++ /dev/null @@ -1,100 +0,0 @@ -"use strict"; -let Promise = require("bluebird"); -let Queue = require("./Queue"); - -// A Queue Pool is a queue which is backed by a pool of queues which can be serviced -// concurrently. The number of items which can be processed concurrently is the size -// of the queue. The QueuePool always operates in a FIFO manner, even when all queues -// are occupied. -class QueuePool { - - // Construct a new Queue Pool. - // This consists of multiple queues. Items will be inserted into - // the first available free queue. If no queue is free, items will - // be put in a FIFO overflow queue. You can also use an index when - // enqueuing to override this behaviour. - constructor(poolSize, fn) { - if (poolSize < 1) { - throw new Error("Pool size must be at least 1"); - } - this.size = poolSize; - this.fn = fn; - this.queues = []; - for (let i = 0; i < poolSize; i++) { - this.queues.push(new Queue(fn)); - } - this.overflow = new Queue(this._overflow.bind(this)); - this._overflowCount = 0; - } - - // Get number of items waiting to be inserted into a queue. - get waitingItems() { return this.overflow.size(); } - - // Add an item to the queue. ID and item are passed directly to the Queue. - // Index is optional and should be between 0 ~ poolSize-1. It determines - // which queue to put the item into, which will bypass the overflow queue. - // Returns: A promise which resolves when the item has been serviced, and - // the promise returned by the queue function has resolved. - enqueue(id, item, index) { - if (index !== undefined) { - if (index >= this.size || index < 0) { - throw new Error(`enqueue: index ${index} is out of bounds`); - } - return this.queues[index].enqueue(id, item); - } - - // no index specified: first free queue gets it. - let queue = this._freeQueue(); - if (queue) { - // push it to the queue pool immediately. - return queue.enqueue(id, item); - } - // All the queues are busy. - // The overflow queue promise is resolved when the item is pushed - // onto the queue pool. We want to return a promise which resolves - // after the item has finished executing on the queue pool, hence - // the promise chain here. - return this.overflow.enqueue(id, { - id: id, - item: item, - }).then((req) => { - return req.p; - }); - } - - // This is called when a request is at the front of the overflow queue. - _overflow(req) { - let queue = this._freeQueue(); - if (queue) { - // cannot return the raw promise else it will be waited on, whereas we want to return - // the actual promise to the caller of QueuePool.enqueue(); so wrap it up in an object. - return Promise.resolve({ - p: queue.enqueue(req.id, req.item) - }); - } - // wait for any queue to become available - let promises = this.queues.map((q) => { - return q.onceFree(); - }); - return Promise.any(promises).then((q) => { - if (q.size() !== 0) { - throw new Error(`QueuePool overflow: starvation. No free queues.`); - } - return { - p: q.enqueue(req.id, req.item), - }; - }) - } - - _freeQueue() { - for (let i = 0; i < this.queues.length; i++) { - if (this.queues[i].size() === 0) { - return this.queues[i]; - } - } - return null; - } - -} - -module.exports = QueuePool; diff --git a/src/util/QueuePool.ts b/src/util/QueuePool.ts new file mode 100644 index 000000000..c2fea9dcd --- /dev/null +++ b/src/util/QueuePool.ts @@ -0,0 +1,122 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import Promise from "bluebird"; +import { Queue, QueueProcessFn, QueueItem } from "./Queue"; + +/** + * A Queue Pool is a queue which is backed by a pool of queues which can be serviced + * concurrently. The number of items which can be processed concurrently is the size + * of the queue. The QueuePool always operates in a FIFO manner, even when all queues + * are occupied. +**/ +export class QueuePool { + private queues: Queue[] = []; + private overflow: Queue; + + /** + * Construct a new Queue Pool. + * This consists of multiple queues. Items will be inserted into + * the first available free queue. If no queue is free, items will + * be put in a FIFO overflow queue. You can also use an index when + * enqueuing to override this behaviour. + */ + constructor(private size: number, fn: QueueProcessFn) { + if (size < 1) { + throw new Error("Pool size must be at least 1"); + } + for (let i = 0; i < size; i++) { + this.queues.push(new Queue(fn)); + } + this.overflow = new Queue((item: unknown) => { + return this.onOverflow(item as QueueItem); + }); + } + + /** + * Get number of items waiting to be inserted into a queue. + */ + get waitingItems() { + return this.overflow.size(); + } + + /** + * Add an item to the queue. ID and item are passed directly to the Queue. + * Index is optional and should be between 0 ~ poolSize-1. It determines + * which queue to put the item into, which will bypass the overflow queue. + * Returns: A promise which resolves when the item has been serviced, and + * the promise returned by the queue function has resolved. + */ + public enqueue(id: string, item: unknown, index?: number) { + if (index !== undefined) { + if (index >= this.size || index < 0) { + throw new Error(`enqueue: index ${index} is out of bounds`); + } + return this.queues[index].enqueue(id, item); + } + + // no index specified: first free queue gets it. + const queue = this.freeQueue(); + if (queue) { + // push it to the queue pool immediately. + return queue.enqueue(id, item); + } + // All the queues are busy. + // The overflow queue promise is resolved when the item is pushed + // onto the queue pool. We want to return a promise which resolves + // after the item has finished executing on the queue pool, hence + // the promise chain here. + return this.overflow.enqueue(id, { + id: id, + item: item, + }).then((req: any) => { + return req.p; + }); + } + + // This is called when a request is at the front of the overflow queue. + private onOverflow(req: QueueItem) { + let queue = this.freeQueue(); + if (queue) { + // cannot return the raw promise else it will be waited on, whereas we want to return + // the actual promise to the caller of QueuePool.enqueue(); so wrap it up in an object. + return Promise.resolve({ + p: queue.enqueue(req.id, req.item) + }); + } + // wait for any queue to become available + const promises = this.queues.map((q) => { + return q.onceFree(); + }); + return Promise.any(promises).then((q) => { + if ((q as Queue).size() !== 0) { + throw new Error(`QueuePool overflow: starvation. No free queues.`); + } + return { + p: q.enqueue(req.id, req.item), + }; + }) + } + + private freeQueue() { + for (let i = 0; i < this.queues.length; i++) { + if (this.queues[i].size() === 0) { + return this.queues[i]; + } + } + return null; + } + +} From 976646dbad69ef73b7ccba6fe9e744763c5f99e0 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 16:58:17 +0100 Subject: [PATCH 084/350] Tweaks --- src/irc/ClientPool.ts | 6 ++++-- src/models/BridgeRequest.ts | 2 +- src/util/Queue.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 239a26993..eab02ac80 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -15,13 +15,15 @@ limitations under the License. */ const stats = require("../config/stats"); -const log = require("../logging").get("ClientPool"); -const { QueuePool } = require("../util/QueuePool"); +import * as logging from "../logging"; +import { QueuePool } from "../util/QueuePool"; import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer } from "../irc/IrcServer"; +const log = logging.get("ClientPool"); + /* * Maintains a lookup of connected IRC clients. These connections are transient * and may be closed for a variety of reasons. diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index 4c57b281b..3e382c69c 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -const logging = require("../logging"); +import logging = require("../logging"); const log = logging.get("req"); export class BridgeRequest { diff --git a/src/util/Queue.ts b/src/util/Queue.ts index e927a6840..ba0993e24 100644 --- a/src/util/Queue.ts +++ b/src/util/Queue.ts @@ -24,7 +24,7 @@ export interface QueueItem { defer: Defer; } -export type QueueProcessFn = (item: unknown) => Bluebird; +export type QueueProcessFn = (item: unknown) => Bluebird|void; export class Queue { private queue: QueueItem[] = []; From dd62d1e7a390005cb227e6a5f95fc6e54c6f171f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 17:25:08 +0100 Subject: [PATCH 085/350] Linting tweaks --- src/irc/ClientPool.ts | 85 +++++++++++++---------- src/models/BridgeRequest.ts | 13 ++-- src/util/Queue.ts | 12 ++-- src/util/QueuePool.ts | 10 +-- types/matrix-appservice-bridge/index.d.ts | 12 ++++ 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index eab02ac80..03d54a1e8 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -14,30 +14,41 @@ See the License for the specific language governing permissions and limitations under the License. */ -const stats = require("../config/stats"); +import * as stats from "../config/stats"; import * as logging from "../logging"; import { QueuePool } from "../util/QueuePool"; import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer } from "../irc/IrcServer"; - +import { AgeCounter, MatrixUser, MatrixRoom } from "matrix-appservice-bridge"; const log = logging.get("ClientPool"); +// We do not have these yet +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type BridgedClient = any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type IrcBridge = any; + +interface ReconnectionItem { + cli: BridgedClient; + chanList: string[]; +} + /* * Maintains a lookup of connected IRC clients. These connections are transient * and may be closed for a variety of reasons. */ export class ClientPool { - private botClients: { [serverDomain: string]: any}; + private botClients: { [serverDomain: string]: BridgedClient}; private virtualClients: { [serverDomain: string]: { - nicks: { [nickname: string]: any}, - userIds: { [userId: string]: any}, - pending: { [nick: string]: any}, - }}; + nicks: { [nickname: string]: BridgedClient}; + userIds: { [userId: string]: BridgedClient}; + pending: { [nick: string]: BridgedClient}; + };}; private virtualClientCounts: { [serverDomain: string]: number }; - private reconnectQueues: { [serverDomain: string]: any }; - constructor(private ircBridge: any) { + private reconnectQueues: { [serverDomain: string]: QueuePool }; + constructor(private ircBridge: IrcBridge) { // The list of bot clients on servers (not specific users) this.botClients = { }; @@ -69,25 +80,25 @@ export class ClientPool { public killAllClients(): Bluebird { const domainList = Object.keys(this.virtualClients); - let clients: any[] = []; + let clients: BridgedClient[] = []; domainList.forEach((domain) => { clients = clients.concat( Object.keys(this.virtualClients[domain].nicks).map( (nick: string) => this.virtualClients[domain].nicks[nick] ) ); - + clients = clients.concat( Object.keys(this.virtualClients[domain].userIds).map( (userId: string) => this.virtualClients[domain].userIds[userId] ) ); - + clients.push(this.botClients[domain]); }); - + clients = clients.filter((c) => Boolean(c)); - + return Bluebird.all( clients.map( (client) => client.kill() @@ -103,9 +114,9 @@ export class ClientPool { if (q === undefined) { q = this.reconnectQueues[server.domain] = new QueuePool( server.getConcurrentReconnectLimit(), - (item: any) => { + (item) => { log.info(`Reconnecting client. ${q.waitingItems} left.`); - return this.reconnectClient(item); + return this.reconnectClient(item as ReconnectionItem); } ); } @@ -113,7 +124,7 @@ export class ClientPool { } - public setBot(server: IrcServer, client: any) { + public setBot(server: IrcServer, client: BridgedClient) { this.botClients[server.domain] = client; } @@ -121,7 +132,7 @@ export class ClientPool { return this.botClients[server.domain]; } - public createIrcClient(ircClientConfig: IrcClientConfig, matrixUser: any, isBot: boolean) { + public createIrcClient(ircClientConfig: IrcClientConfig, matrixUser: MatrixUser, isBot: boolean) { const bridgedClient = this.ircBridge.createBridgedClient( ircClientConfig, matrixUser, isBot ); @@ -193,9 +204,9 @@ export class ClientPool { return cli; } - public getBridgedClientsForUserId(userId: string) { + public getBridgedClientsForUserId(userId: string): BridgedClient[] { const domainList = Object.keys(this.virtualClients); - const clientList: any[] = []; + const clientList: BridgedClient[] = []; domainList.forEach((domain) => { const cli = this.virtualClients[domain].userIds[userId]; if (cli && !cli.isDead()) { @@ -208,7 +219,7 @@ export class ClientPool { public getBridgedClientsForRegex(userIdRegexString: string) { const userIdRegex = new RegExp(userIdRegexString); const domainList = Object.keys(this.virtualClients); - const clientList: {[userId: string]: any} = {}; + const clientList: {[userId: string]: BridgedClient} = {}; domainList.forEach((domain) => { Object.keys( this.virtualClients[domain].userIds @@ -248,7 +259,7 @@ export class ClientPool { ); // find the oldest client to kill. - let oldest: any = null; + let oldest: BridgedClient|null = null; Object.keys(this.virtualClients[server.domain].nicks).forEach((nick: string) => { const client = this.virtualClients[server.domain].nicks[nick]; if (!client) { @@ -286,7 +297,7 @@ export class ClientPool { let count = 0; Object.keys(this.virtualClients).forEach((domain) => { - let server = this.ircBridge.getServer(domain); + const server = this.ircBridge.getServer(domain); count += this.getNumberOfConnections(server); }); @@ -300,7 +311,7 @@ export class ClientPool { return 0; } - public updateActiveConnectionMetrics(serverDomain: string, ageCounter: any): void { + public updateActiveConnectionMetrics(serverDomain: string, ageCounter: AgeCounter): void { if (this.virtualClients[serverDomain] === undefined) { return; } @@ -337,7 +348,7 @@ export class ClientPool { stats.ircClients(server.domain, this.getNumberOfConnections(server)); } - private removeBridgedClient(bridgedClient: any): void { + private removeBridgedClient(bridgedClient: BridgedClient): void { const server = bridgedClient.server; this.virtualClients[server.domain].userIds[bridgedClient.userId] = undefined; this.virtualClients[server.domain].nicks[bridgedClient.nick] = undefined; @@ -348,7 +359,7 @@ export class ClientPool { } } - private onClientConnected(bridgedClient: any): void { + private onClientConnected(bridgedClient: BridgedClient): void { const server = bridgedClient.server; const oldNick = bridgedClient.nick; const actualNick = bridgedClient.unsafeClient.nick; @@ -366,7 +377,7 @@ export class ClientPool { } } - private onClientDisconnected(bridgedClient: any): void { + private onClientDisconnected(bridgedClient: BridgedClient): void { this.removeBridgedClient(bridgedClient); this.sendConnectionMetric(bridgedClient.server); @@ -408,7 +419,7 @@ export class ClientPool { log.info(`Dropping ${cli._id} ${cli.nick} because they are not joined to any channels`); return; } - let queue = this.getOrCreateReconnectQueue(cli.server); + const queue = this.getOrCreateReconnectQueue(cli.server); if (queue === null) { this.reconnectClient({ cli: cli, @@ -422,7 +433,7 @@ export class ClientPool { }); } - private reconnectClient(cliChan: any): void { + private reconnectClient(cliChan: ReconnectionItem): void { const cli = cliChan.cli; const chanList: string[] = cliChan.chanList; return cli.connect().then(() => { @@ -434,19 +445,19 @@ export class ClientPool { cli.joinChannel(c); }); this.sendConnectionMetric(cli.server); - }, (e: Error) => { + }, () => { log.error( "<%s> Failed to reconnect %s@%s", cli._id, cli.nick, cli.server.domain ); }); } - private onNickChange(bridgedClient: any, oldNick: string, newNick: string): void { + private onNickChange(bridgedClient: BridgedClient, oldNick: string, newNick: string): void { this.virtualClients[bridgedClient.server.domain].nicks[oldNick] = undefined; this.virtualClients[bridgedClient.server.domain].nicks[newNick] = bridgedClient; } - private async onJoinError (bridgedClient: any, chan: string, err: string): Promise { + private async onJoinError (bridgedClient: BridgedClient, chan: string, err: string): Promise { const errorsThatShouldKick = [ "err_bannedfromchan", // they aren't allowed in channels they are banned on. "err_inviteonlychan", // they aren't allowed in invite only channels @@ -463,10 +474,10 @@ export class ClientPool { // TODO: this is a bit evil, no one in their right mind would expect // the client pool to be kicking matrix users from a room :( log.info(`Kicking ${bridgedClient.userId} from room due to ${err}`); - let matrixRooms = await this.ircBridge.getStore().getMatrixRoomsForChannel( + const matrixRooms = await this.ircBridge.getStore().getMatrixRoomsForChannel( bridgedClient.server, chan ); - let promises = matrixRooms.map((room: any) => { + const promises = matrixRooms.map((room: MatrixRoom) => { return this.ircBridge.getAppServiceBridge().getIntent().kick( room.getId(), bridgedClient.userId, `IRC error on ${chan}: ${err}` ); @@ -474,11 +485,11 @@ export class ClientPool { await Promise.all(promises); } - private onNames(bridgedClient: any, chan: string, names: any): Bluebird { - let mls = this.ircBridge.memberListSyncers[bridgedClient.server.domain]; + private onNames(bridgedClient: BridgedClient, chan: string, names: {[key: string]: string}): Bluebird { + const mls = this.ircBridge.memberListSyncers[bridgedClient.server.domain]; if (!mls) { return Bluebird.resolve(); } return mls.updateIrcMemberList(chan, names); } -} \ No newline at end of file +} diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index 3e382c69c..10649dfba 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -15,11 +15,14 @@ limitations under the License. */ import logging = require("../logging"); +import { Request } from "matrix-appservice-bridge"; const log = logging.get("req"); +type Logger = any; + export class BridgeRequest { - private log: any; - constructor(private req: any) { + private log: Logger; + constructor(private req: Request) { const isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); } @@ -28,15 +31,15 @@ export class BridgeRequest { return this.req.getPromise(); } - resolve(thing: any) { + resolve(thing?: unknown) { this.req.resolve(thing); } - reject(err: any) { + reject(err?: unknown) { this.req.reject(err); } public static ERR_VIRTUAL_USER = "virtual-user"; public static ERR_NOT_MAPPED = "virtual-user"; public static ERR_DROPPED = "virtual-user"; -} \ No newline at end of file +} diff --git a/src/util/Queue.ts b/src/util/Queue.ts index ba0993e24..9bcf72862 100644 --- a/src/util/Queue.ts +++ b/src/util/Queue.ts @@ -65,20 +65,20 @@ export class Queue { */ public size(): number { return this.queue.length + (this.processing ? 1 : 0); - }; + } /** * Return a promise which is resolved when this queue is free (0 items in queue). * @return {Promise} Resolves to the Queue itself. */ - public onceFree(): Promise { + public onceFree(): Promise { if (this.size() === 0) { return Promise.resolve(); } const defer = promiseutil.defer(); this.onceFreeDefers.push(defer); return defer.promise; - }; + } private fireOnceFree() { this.onceFreeDefers.forEach((d) => { @@ -115,7 +115,7 @@ export class Queue { }); } return defer.promise; - }; + } private retry () { setTimeout(this.consume.bind(this), this.intervalMs); @@ -134,8 +134,8 @@ export class Queue { return; } try { - let thing = this.processFn(this.processing.item); - let result = yield thing; + const thing = this.processFn(this.processing.item); + const result = yield thing; this.processing.defer.resolve(result); } catch (err) { diff --git a/src/util/QueuePool.ts b/src/util/QueuePool.ts index c2fea9dcd..c0e667560 100644 --- a/src/util/QueuePool.ts +++ b/src/util/QueuePool.ts @@ -16,7 +16,7 @@ limitations under the License. import Promise from "bluebird"; import { Queue, QueueProcessFn, QueueItem } from "./Queue"; -/** +/** * A Queue Pool is a queue which is backed by a pool of queues which can be serviced * concurrently. The number of items which can be processed concurrently is the size * of the queue. The QueuePool always operates in a FIFO manner, even when all queues @@ -81,14 +81,14 @@ export class QueuePool { return this.overflow.enqueue(id, { id: id, item: item, - }).then((req: any) => { - return req.p; + }).then((req) => { + return (req as {p: Promise}).p; }); } // This is called when a request is at the front of the overflow queue. private onOverflow(req: QueueItem) { - let queue = this.freeQueue(); + const queue = this.freeQueue(); if (queue) { // cannot return the raw promise else it will be waited on, whereas we want to return // the actual promise to the caller of QueuePool.enqueue(); so wrap it up in an object. @@ -98,7 +98,7 @@ export class QueuePool { } // wait for any queue to become available const promises = this.queues.map((q) => { - return q.onceFree(); + return q.onceFree().then(() => q); }); return Promise.any(promises).then((q) => { if ((q as Queue).size() !== 0) { diff --git a/types/matrix-appservice-bridge/index.d.ts b/types/matrix-appservice-bridge/index.d.ts index a4401628d..9b6afc1b3 100644 --- a/types/matrix-appservice-bridge/index.d.ts +++ b/types/matrix-appservice-bridge/index.d.ts @@ -135,4 +135,16 @@ declare module 'matrix-appservice-bridge' { unlinkUserIds (matrixUserId: string, remoteUserId: string): Promise unlinkUsers (matrixUser: MatrixUser, remoteUser: RemoteUser): Promise } + + export class AgeCounter { + bump (ageInSec: number): void; + } + + export class Request { + getData(): any; + getPromise(): Promise; + getId(): string; + resolve(item: unknown): void; + reject(err: unknown): void; + } } \ No newline at end of file From 00bbaa081c111f2d15cccd534c38e33c5fafb2ce Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 17:30:39 +0100 Subject: [PATCH 086/350] Newsfile --- changelog.d/826.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/826.misc diff --git a/changelog.d/826.misc b/changelog.d/826.misc new file mode 100644 index 000000000..8a33f699c --- /dev/null +++ b/changelog.d/826.misc @@ -0,0 +1 @@ +Convert ClientPool and associated dependencies to Typescript \ No newline at end of file From d3f84f7368005fa93e5d6d1621580bddef476e35 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 17:31:39 +0100 Subject: [PATCH 087/350] Lint --- src/models/BridgeRequest.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index 10649dfba..f76b00885 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -18,6 +18,8 @@ import logging = require("../logging"); import { Request } from "matrix-appservice-bridge"; const log = logging.get("req"); +// We do not have types for logging yet. +// eslint-disable-next-line @typescript-eslint/no-explicit-any type Logger = any; export class BridgeRequest { From ee2dd6279219c6e0a24eab670f883e2751b995a0 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 17:34:15 +0100 Subject: [PATCH 088/350] Lint fix --- src/irc/Ident.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index a4435aec3..99027ed42 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -135,7 +135,8 @@ class IdentSrv { if (username) { log.debug("Port %s is %s", localPort, username); response = `${localPort},${remotePort}:USERID:UNIX:${username}\r\n`; - } else { + } + else { log.debug("No user on port %s", localPort); response = `${localPort},${remotePort}:ERROR:NO-USER\r\n`; } From 28eeb45c57a89562554b6d4a96c7466584c6511c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 18:28:29 +0100 Subject: [PATCH 089/350] Convert logging to Typescript --- src/logging.js | 225 ----------------------------------------- src/logging.ts | 267 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+), 225 deletions(-) delete mode 100644 src/logging.js create mode 100644 src/logging.ts diff --git a/src/logging.js b/src/logging.js deleted file mode 100644 index ff9a024a2..000000000 --- a/src/logging.js +++ /dev/null @@ -1,225 +0,0 @@ -/* - * This module provides python-like logging capabilities using winston. - */ - -const winston = require("winston"); -require('winston-daily-rotate-file'); - -let loggerConfig = { - level: "debug", //debug|info|warn|error - logfile: undefined, // path to file - errfile: undefined, // path to file - toConsole: true, // make a console logger - maxFiles: 5, - verbose: false -}; - -const loggers = { - // name_of_logger: Logger -}; -let loggerTransports; // from config - -const UNCAUGHT_EXCEPTION_ERRCODE = 101; - -const makeTransports = function() { - var timestampFn = function() { - return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); - }; - var formatterFn = function(opts) { - return opts.timestamp() + ' ' + - opts.level.toUpperCase() + ':' + - (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + - (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + - (opts.meta && opts.meta.dir ? opts.meta.dir : "") + - (undefined !== opts.message ? opts.message : ''); - }; - - var transports = []; - if (loggerConfig.toConsole) { - transports.push(new (winston.transports.Console)({ - json: false, - name: "console", - timestamp: timestampFn, - formatter: formatterFn, - level: loggerConfig.level - })); - } - if (loggerConfig.logfile) { - transports.push(new (winston.transports.DailyRotateFile)({ - filename: loggerConfig.logfile, - json: false, - name: "logfile", - level: loggerConfig.level, - timestamp: timestampFn, - formatter: formatterFn, - maxFiles: loggerConfig.maxFiles, - datePattern: "YYYY-MM-DD", - tailable: true - })); - } - if (loggerConfig.errfile) { - transports.push(new (winston.transports.DailyRotateFile)({ - filename: loggerConfig.errfile, - json: false, - name: "errorfile", - level: "error", - timestamp: timestampFn, - formatter: formatterFn, - maxFiles: loggerConfig.maxFiles, - datePattern: "YYYY-MM-DD", - tailable: true - })); - } - // by default, EventEmitters will whine if you set more than 10 listeners on - // them. The 'transport' is an emitter which the loggers listen for errors - // from. Since we have > 10 files (each with their own logger), we get - // warnings. Set the max listeners to unlimited to suppress the warning. - transports.forEach(function(transport) { - transport.setMaxListeners(0); - }); - return transports; -}; - -const createLogger = function(nameOfLogger) { - // lazily load the transports if one wasn't set from configure() - if (!loggerTransports) { - loggerTransports = makeTransports(); - } - - return new (winston.Logger)({ - transports: loggerTransports, - // winston doesn't support getting the logger category from the - // formatting function, which is a shame. Instead, write a rewriter - // which sets the 'meta' info for the logged message with the loggerName - rewriters: [ - function(level, msg, meta) { - if (!meta) { meta = {}; } - meta.loggerName = nameOfLogger; - return meta; - } - ] - }); -}; - -module.exports = { - /* - * Obtain a logger by name, creating one if necessary. - */ - get: function(nameOfLogger) { - if (loggers[nameOfLogger]) { - return loggers[nameOfLogger]; - } - var logger = createLogger(nameOfLogger); - loggers[nameOfLogger] = logger; - logger.logErr = function(e) { - logger.error("Error: %s", JSON.stringify(e)); - if (e.stack) { - logger.error(e.stack); - } - }; - return logger; - }, - - /* - * Configure how loggers should be created. - */ - configure: function(opts) { - if (!opts) { - return; - } - loggerConfig = opts; - loggerTransports = makeTransports(); - // reconfigure any existing loggers. They may have been lazily loaded - // with the default config, which is now being overwritten by this - // configure() call. - Object.keys(loggers).forEach(function(loggerName) { - var existingLogger = loggers[loggerName]; - // remove each individual transport - var transportNames = ["logfile", "console", "errorfile"]; - transportNames.forEach(function(tname) { - if (existingLogger.transports[tname]) { - existingLogger.remove(tname); - } - }); - // apply the new transports - loggerTransports.forEach(function(transport) { - existingLogger.add(transport, undefined, true); - }); - }); - }, - - isVerbose: function() { - return loggerConfig.verbose; - }, - - newRequestLogger: function(baseLogger, requestId, isFromIrc) { - var decorate = function(fn, args) { - var newArgs = []; - // don't slice this; screws v8 optimisations apparently - for (var i = 0; i < args.length; i++) { - newArgs.push(args[i]); - } - // add a piece of metadata to the log line, with the request ID. - newArgs[args.length] = { - reqId: requestId, - dir: (isFromIrc ? "[I->M] " : "[M->I] ") - }; - fn.apply(baseLogger, newArgs); - }; - - return { - debug: function() { decorate(baseLogger.debug, arguments); }, - info: function() { decorate(baseLogger.info, arguments); }, - warn: function() { decorate(baseLogger.warn, arguments); }, - error: function() { decorate(baseLogger.error, arguments); }, - log: function() { decorate(baseLogger.log, arguments); } - }; - }, - - setUncaughtExceptionLogger: function(exceptionLogger) { - process.on("uncaughtException", function(e) { - // Log to stderr first and foremost, to avoid any chance of us missing a flush. - console.error("FATAL EXCEPTION"); - console.error(e && e.stack ? e.stack : String(e)); - - // Log to winston handlers afterwards, if we can. - exceptionLogger.error("FATAL EXCEPTION"); - if (e && e.stack) { - exceptionLogger.error(e.stack); - } - else { - exceptionLogger.error(e); - } - - // We exit with UNCAUGHT_EXCEPTION_ERRCODE to ensure that the poor - // developers debugging the bridge can identify where it exploded. - - // There have been issues where winston has failed to log the last - // few lines before quitting, which I suspect is due to it not flushing. - // Since we know we're going to die at this point, log something else - // and forcibly flush all the transports before exiting. - exceptionLogger.error("Terminating (exitcode=1)", function(err) { - var numFlushes = 0; - var numFlushed = 0; - Object.keys(exceptionLogger.transports).forEach(function(k) { - if (exceptionLogger.transports[k]._stream) { - numFlushes += 1; - exceptionLogger.transports[k]._stream.once("finish", function() { - numFlushed += 1; - if (numFlushes === numFlushed) { - process.exit(UNCAUGHT_EXCEPTION_ERRCODE); - } - }); - exceptionLogger.transports[k]._stream.on("error", function() { - // swallow - }); - exceptionLogger.transports[k]._stream.end(); - } - }); - if (numFlushes === 0) { - process.exit(UNCAUGHT_EXCEPTION_ERRCODE); - } - }); - }); - } -}; diff --git a/src/logging.ts b/src/logging.ts new file mode 100644 index 000000000..581a6bac0 --- /dev/null +++ b/src/logging.ts @@ -0,0 +1,267 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +/* + * This module provides python-like logging capabilities using winston. + */ + +import winston, { TransportInstance, LeveledLogMethod, LoggerInstance } from "winston"; +import "winston-daily-rotate-file"; +import { WriteStream } from "fs"; + + +interface FormatterFnOpts { + timestamp: () => string; + level: string; + meta: {[key: string]: string}; + message: string; +} + +interface LoggerConfig { + level: "debug"|"info"|"warn"|"error"; + logfile?: string; // path to file + errfile?: string; // path to file + toConsole: boolean; + maxFiles: number; + verbose: boolean; +} + +const UNCAUGHT_EXCEPTION_ERRCODE = 101; + +let loggerConfig: LoggerConfig = { + level: "debug", //debug|info|warn|error + logfile: undefined, // path to file + errfile: undefined, // path to file + toConsole: true, // make a console logger + maxFiles: 5, + verbose: false +}; + +const loggers: {[name: string]: LoggerInstance } = { + // name_of_logger: Logger +}; + +let loggerTransports: TransportInstance[]; // from config + +const makeTransports = function() { + const timestampFn = function() { + return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); + }; + const formatterFn = function(opts: FormatterFnOpts) { + return opts.timestamp() + ' ' + + opts.level.toUpperCase() + ':' + + (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + + (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + + (opts.meta && opts.meta.dir ? opts.meta.dir : "") + + (undefined !== opts.message ? opts.message : ''); + }; + + let transports = []; + if (loggerConfig.toConsole) { + transports.push(new (winston.transports.Console)({ + json: false, + name: "console", + timestamp: timestampFn, + formatter: formatterFn, + level: loggerConfig.level + })); + } + if (loggerConfig.logfile) { + transports.push(new (winston.transports.DailyRotateFile)({ + filename: loggerConfig.logfile, + json: false, + name: "logfile", + level: loggerConfig.level, + timestamp: timestampFn, + formatter: formatterFn, + maxFiles: loggerConfig.maxFiles, + datePattern: "YYYY-MM-DD", + tailable: true + })); + } + if (loggerConfig.errfile) { + transports.push(new (winston.transports.DailyRotateFile)({ + filename: loggerConfig.errfile, + json: false, + name: "errorfile", + level: "error", + timestamp: timestampFn, + formatter: formatterFn, + maxFiles: loggerConfig.maxFiles, + datePattern: "YYYY-MM-DD", + tailable: true + })); + } + // by default, EventEmitters will whine if you set more than 10 listeners on + // them. The 'transport' is an emitter which the loggers listen for errors + // from. Since we have > 10 files (each with their own logger), we get + // warnings. Set the max listeners to unlimited to suppress the warning. + transports.forEach(function(transport) { + transport.setMaxListeners(0); + }); + return transports; +}; + +const createLogger = function(nameOfLogger: string) { + // lazily load the transports if one wasn't set from configure() + if (!loggerTransports) { + loggerTransports = makeTransports(); + } + + return new (winston.Logger)({ + transports: loggerTransports, + // winston doesn't support getting the logger category from the + // formatting function, which is a shame. Instead, write a rewriter + // which sets the 'meta' info for the logged message with the loggerName + rewriters: [ + function(level, msg, meta) { + if (!meta) { meta = {}; } + meta.loggerName = nameOfLogger; + return meta; + } + ] + }); +}; + +/** + * Obtain a logger by name, creating one if necessary. + */ +export function get(nameOfLogger: string) { + if (loggers[nameOfLogger]) { + return loggers[nameOfLogger]; + } + let logger = createLogger(nameOfLogger); + loggers[nameOfLogger] = logger; + const ircLogger = { + logErr: (e: Error) => { + logger.error("Error: %s", JSON.stringify(e)); + if (e.stack) { + logger.error(e.stack); + } + }, + ...logger, + } + return ircLogger; +} + +export const getLogger = get; + +/** + * Configure how loggers should be created. + */ +export function configure(opts: LoggerConfig) { + if (!opts) { + return; + } + loggerConfig = opts; + loggerTransports = makeTransports(); + // reconfigure any existing loggers. They may have been lazily loaded + // with the default config, which is now being overwritten by this + // configure() call. + Object.keys(loggers).forEach(function(loggerName) { + let existingLogger = loggers[loggerName]; + // remove each individual transport + let transportNames = ["logfile", "console", "errorfile"]; + transportNames.forEach(function(tname) { + if (existingLogger.transports[tname]) { + existingLogger.remove(tname); + } + }); + // apply the new transports + loggerTransports.forEach(function(transport) { + existingLogger.add(transport, undefined, true); + }); + }); +} + +export function isVerbose() { + return loggerConfig.verbose; +} + +export function newRequestLogger(baseLogger: LoggerInstance, requestId: string, isFromIrc: boolean) { + const decorate = function(fn: LeveledLogMethod, args: IArguments ) { + let newArgs: Array = []; + // don't slice this; screws v8 optimisations apparently + for (let i = 0; i < args.length; i++) { + newArgs.push(args[i]); + } + // add a piece of metadata to the log line, with the request ID. + newArgs[args.length] = { + reqId: requestId, + dir: (isFromIrc ? "[I->M] " : "[M->I] ") + }; + // Typescript doesn't like us mangling args like this, but we have to. + // @ts-ignore + fn.apply(baseLogger, newArgs); + }; + return { + debug: function() { decorate(baseLogger.debug, arguments); }, + info: function() { decorate(baseLogger.info, arguments); }, + warn: function() { decorate(baseLogger.warn, arguments); }, + error: function() { decorate(baseLogger.error, arguments); }, + }; +} + +export function setUncaughtExceptionLogger(exceptionLogger: LoggerInstance) { + process.on("uncaughtException", function(e) { + // Log to stderr first and foremost, to avoid any chance of us missing a flush. + console.error("FATAL EXCEPTION"); + console.error(e && e.stack ? e.stack : String(e)); + + // Log to winston handlers afterwards, if we can. + exceptionLogger.error("FATAL EXCEPTION"); + if (e && e.stack) { + exceptionLogger.error(e.stack); + } + else { + exceptionLogger.error(e.name, e.message); + } + + // We exit with UNCAUGHT_EXCEPTION_ERRCODE to ensure that the poor + // developers debugging the bridge can identify where it exploded. + + // There have been issues where winston has failed to log the last + // few lines before quitting, which I suspect is due to it not flushing. + // Since we know we're going to die at this point, log something else + // and forcibly flush all the transports before exiting. + exceptionLogger.error("Terminating (exitcode=1)", function(err: Error) { + let numFlushes = 0; + let numFlushed = 0; + Object.keys(exceptionLogger.transports).forEach(function(k) { + // We need to access the unexposed _stream + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const stream: WriteStream = (exceptionLogger.transports[k] as any)._stream; + if (stream) { + numFlushes += 1; + stream.once("finish", function() { + numFlushed += 1; + if (numFlushes === numFlushed) { + process.exit(UNCAUGHT_EXCEPTION_ERRCODE); + } + }); + stream.on("error", function() { + // swallow + }); + stream.end(); + } + }); + if (numFlushes === 0) { + process.exit(UNCAUGHT_EXCEPTION_ERRCODE); + } + }); + }); +} From 65c09bf9773929c44438b70bd070c56c1d58c590 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 18:28:50 +0100 Subject: [PATCH 090/350] Change logging imports --- src/datastore/StringCrypto.ts | 4 ++-- src/datastore/postgres/PgDataStore.ts | 4 ++-- src/irc/IrcServer.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/datastore/StringCrypto.ts b/src/datastore/StringCrypto.ts index a6c0106b2..793b819dc 100644 --- a/src/datastore/StringCrypto.ts +++ b/src/datastore/StringCrypto.ts @@ -16,9 +16,9 @@ limitations under the License. import * as crypto from "crypto"; import * as fs from "fs"; -import * as logging from "../logging"; +import { getLogger } from "../logging"; -const log = logging.get("CryptoStore"); +const log = getLogger("CryptoStore"); export class StringCrypto { private privateKey!: string; diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 888866364..8c4c05f08 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -24,13 +24,13 @@ import { IrcRoom } from "../../models/IrcRoom"; import { IrcClientConfig } from "../../models/IrcClientConfig"; import { IrcServer, IrcServerConfig } from "../../irc/IrcServer"; -import * as logging from "../../logging"; +import { getLogger } from "../../logging"; import Bluebird from "bluebird"; import { stat } from "fs"; import { StringCrypto } from "../StringCrypto"; import { toIrcLowerCase } from "../../irc/formatting"; -const log = logging.get("PgDatastore"); +const log = getLogger("PgDatastore"); export class PgDataStore implements DataStore { private serverMappings: {[domain: string]: IrcServer} = {}; diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 37190cbba..8430d7222 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import * as logging from "../logging"; +import { getLogger } from "../logging"; import * as BridgedClient from "./BridgedClient"; import { IrcClientConfig } from "../models/IrcClientConfig"; -const log = logging.get("IrcServer"); +const log = getLogger("IrcServer"); const GROUP_ID_REGEX = /^\+\S+:\S+$/ type MembershipSyncKind = "incremental"|"initial"; From 46a057a258a610dd1c962b334c2b3912717ae67c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Tue, 1 Oct 2019 18:32:27 +0100 Subject: [PATCH 091/350] Newsfile --- changelog.d/827.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/827.misc diff --git a/changelog.d/827.misc b/changelog.d/827.misc new file mode 100644 index 000000000..7cc2c6d95 --- /dev/null +++ b/changelog.d/827.misc @@ -0,0 +1 @@ +Convert logging to Typescript \ No newline at end of file From 046735f2d00e14585ad3b69de7da418e938cf5cf Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 11:54:46 +0100 Subject: [PATCH 092/350] Add postinstall hook --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 66935625b..fdff334e7 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "node": ">=6.9" }, "scripts": { + "postinstall": "npm run build", "build": "tsc --project ./tsconfig.json", "test": "BLUEBIRD_DEBUG=1 node --max_old_space_size=3072 node_modules/jasmine/bin/jasmine.js --stop-on-failure=true", "lint:js": "eslint --max-warnings 0 src/**/*.js app.js spec", From ac1628c801a60c0fe323f4a41936d36724ad2522 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 11:55:00 +0100 Subject: [PATCH 093/350] Fix error caused by using NeDB --- src/main.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 40ec5a339..da0041fcf 100644 --- a/src/main.js +++ b/src/main.js @@ -116,18 +116,21 @@ module.exports.runBridge = Promise.coroutine(function*(port, config, reg, isDBIn // run the bridge const ircBridge = new IrcBridge(config, reg); - + const engine = config.database ? config.database.engine : "nedb"; // Use in-memory DBs if (isDBInMemory) { ircBridge._bridge.opts.roomStore = new RoomBridgeStore(new Datastore()); ircBridge._bridge.opts.userStore = new UserBridgeStore(new Datastore()); } - else if (config.database && config.database.engine === "postgres") { + else if (engine === "postgres") { // Enforce these not to be created ircBridge._bridge.opts.roomStore = undefined; ircBridge._bridge.opts.userStore = undefined; } - else if (config.database) { + else if (engine === "nedb") { + // do nothing. + } + else { throw Error("Invalid database configuration"); } From 88a85ee7de9d225464d277bc5737e2507a75ad51 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 17:12:08 +0100 Subject: [PATCH 094/350] Convert DebugApi to Typescript --- src/DebugApi.js | 414 -------------------------------------------- src/DebugApi.ts | 451 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 451 insertions(+), 414 deletions(-) delete mode 100644 src/DebugApi.js create mode 100644 src/DebugApi.ts diff --git a/src/DebugApi.js b/src/DebugApi.js deleted file mode 100644 index 311283eae..000000000 --- a/src/DebugApi.js +++ /dev/null @@ -1,414 +0,0 @@ -/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping -"use strict"; -const querystring = require("querystring"); -const Promise = require("bluebird"); -const { BridgeRequest } = require("./models/BridgeRequest"); -const log = require("./logging").get("DebugApi"); -const http = require("http"); - -function DebugApi(ircBridge, port, servers, pool, token) { - this.ircBridge = ircBridge; - this.port = port; - this.pool = pool; - this.servers = servers; - this.token = token; -} - -DebugApi.prototype._getClient = function(server, user) { - if (!user) { - return this.pool.getBot(server); - } - return this.pool.getBridgedClientByUserId(server, user); -}; - -DebugApi.prototype.getClientState = function(server, user) { - log.debug("getClientState(%s,%s)", server.domain, user); - let client = this._getClient(server, user); - if (!client) { - return "User " + user + " does not have a client on " + server.domain; - } - return require("util").inspect(client, {colors:true, depth:7}); -}; - -DebugApi.prototype.killUser = function(userId, reason) { - const req = new BridgeRequest(this.ircBridge._bridge.getRequestFactory().newRequest()); - const clients = this.pool.getBridgedClientsForUserId(userId); - return this.ircBridge.matrixHandler.quitUser(req, userId, clients, null, reason); -}; - -// returns a promise to allow a response buffer to be populated -DebugApi.prototype.sendIRCCommand = function(server, user, body) { - log.debug("sendIRCCommand(%s,%s,%s)", server.domain, user, body); - let client = this._getClient(server, user); - if (!client) { - return Promise.resolve( - "User " + user + " does not have a client on " + server.domain + "\n" - ); - } - if (!client.unsafeClient) { - return Promise.resolve( - "There is no underlying client instance.\n" - ); - } - - // store all received response strings - let buffer = []; - let listener = function(msg) { - buffer.push(JSON.stringify(msg)); - } - - client.unsafeClient.on("raw", listener); - // turn rn to n so if there are any new lines they are all n. - body = body.replace("\r\n", "\n"); - body.split("\n").forEach((c) => { - // IRC protocol require rn - client.unsafeClient.conn.write(c + "\r\n"); - buffer.push(c); - }); - - // wait 3s to pool responses - return Promise.delay(3000).then(function() { - // unhook listener to avoid leaking - if (client.unsafeClient) { - client.unsafeClient.removeListener("raw", listener); - } - return buffer.join("\n") + "\n"; - }); -} - -DebugApi.prototype.run = function() { - log.info("DEBUG API LISTENING ON :%d", this.port); - - http.createServer((req, response) => { - try { - let reqPath = req.url.split("?"); - let path = reqPath[0]; - let query = querystring.parse(reqPath[1]); - log.debug(req.method + " " + path); - - if (query["access_token"] !== this.token) { - response.writeHead(403, {"Content-Type": "text/plain"}); - response.write("Invalid or missing ?access_token=. " + - "The app service token is required from the registration.\n"); - response.end(); - log.warn("Failed attempt with token " + query["access_token"]); - return; - } - - if (path == "/killUser") { - let body = ""; - req.on("data", function(chunk) { - body += chunk; - }); - req.on("end", () => { - let promise = null; - try { - body = JSON.parse(body); - if (!body.user_id || !body.reason) { - promise = Promise.reject(new Error("Need user_id and reason")); - } - else { - promise = this.killUser(body.user_id, body.reason); - } - } - catch (err) { - promise = Promise.reject(err); - } - - promise.then(function(r) { - response.writeHead(200, {"Content-Type": "text/plain"}); - response.write(r + "\n"); - response.end(); - }, function(err) { - log.error(err.stack); - response.writeHead(500, {"Content-Type": "text/plain"}); - response.write(err + "\n"); - response.end(); - }); - }); - return; - } - else if (req.method === "POST" && path == "/reapUsers") { - const msgCb = (msg) => { - if (!response.headersSent) { - response.writeHead(200, {"Content-Type": "text/plain"}); - } - response.write(msg + "\n") - } - this.ircBridge.connectionReap( - msgCb, query["server"], parseInt(query["since"]), query["reason"] - ).catch((err) => { - log.error(err.stack); - if (!response.headersSent) { - response.writeHead(500, {"Content-Type": "text/plain"}); - } - response.write(err + "\n"); - }).finally(() => { - response.end(); - }); - return; - } - else if (req.method === "POST" && path == "/killPortal") { - this.killPortal(req, response); - return; - } - else if (req.method === "GET" && path === "/inspectUsers") { - this.inspectUsers(query["regex"], response); - return; - } - - // Looks like /irc/$domain/user/$user_id - let segs = path.split("/"); - if (segs.length !== 5 || segs[1] !== "irc" || segs[3] !== "user") { - response.writeHead(404, {"Content-Type": "text/plain"}); - response.write("Not a valid debug path.\n"); - response.end(); - return; - } - - let domain = segs[2]; - let user = segs[4]; - - log.debug("Domain: %s User: %s", domain, user); - - let server = null; - for (var i = 0; i < this.servers.length; i++) { - if (this.servers[i].domain === domain) { - server = this.servers[i]; - break; - } - } - if (server === null) { - response.writeHead(400, {"Content-Type": "text/plain"}); - response.write("Not a valid domain.\n"); - response.end(); - return; - } - - let body = ""; - req.on("data", function(chunk) { - body += chunk; - }); - - req.on("end", () => { - // Create a promise which resolves to a response string - let promise = null; - if (req.method === "GET") { - try { - let resBody = this.getClientState(server, user); - if (!resBody.endsWith("\n")) { - resBody += "\n"; - } - promise = Promise.resolve(resBody); - } - catch (err) { - promise = Promise.reject(err); - } - } - else if (req.method === "POST") { - promise = this.sendIRCCommand(server, user, body) - } - else { - promise = Promise.reject(new Error("Bad HTTP method")); - } - - promise.done(function(r) { - response.writeHead(200, {"Content-Type": "text/plain"}); - response.write(r); - response.end(); - }, function(err) { - log.error(err.stack); - response.writeHead(500, {"Content-Type": "text/plain"}); - response.write(err + "\n"); - response.end(); - }); - }); - } - catch (err) { - log.error(err.stack); - } - }).listen(this.port); -} - -DebugApi.prototype.killPortal = Promise.coroutine(function*(req, response) { - const result = { - error: [], // string|[string] containing a fatal error or minor errors. - stages: [] // stages completed for removing the room. It's possible it might only - // half complete, and we should make that obvious. - }; - const body = yield this._wrapJsonReq(req, response); - - // Room room_id to lookup and delete the alias from. - const roomId = body["room_id"]; - - // IRC server domain - const domain = body["domain"]; - - // IRC channel - const channel = body["channel"]; - - // Should we tell the room about the deletion. Defaults to true. - const notice = !(body["leave_notice"] === false); - - // Should we remove the alias from the room. Defaults to true. - const remove_alias = !(body["remove_alias"] === false); - - // These keys are required. - ["room_id", "channel", "domain"].forEach((key) => { - if (typeof(body[key]) !== "string") { - result.error.push(`'${key}' is missing from body or not a string`); - } - }); - if (result.error.length > 0) { - this._wrapJsonResponse(result.error, false, response); - return; - } - - log.warn( -`Requested deletion of portal room alias ${roomId} through debug API -Domain: ${domain} -Channel: ${channel} -Leave Notice: ${notice} -Remove Alias: ${remove_alias}`); - - // Find room - let room = yield this.ircBridge.getStore().getRoom( - roomId, - domain, - channel, - "alias" - ); - if (room === null) { - result.error = "Room not found"; - this._wrapJsonResponse(result, false, response); - return; - } - - const server = this.servers.find((srv) => srv.domain === domain); - if (server === null) { - result.error = "Server not found!"; - this._wrapJsonResponse(result, false, response); - return; - } - - // Drop room from room store. - yield this.ircBridge.getStore().removeRoom( - roomId, - domain, - channel, - "alias" - ); - result.stages.push("Removed room from store"); - - if (notice) { - try { - yield this.ircBridge.getAppServiceBridge().getIntent().sendEvent(roomId, "notice", - { - body: `This room has been unbridged from ${channel} (${server.getReadableName()})` - }); - result.stages.push("Left notice in room"); - } - catch (e) { - result.error.push("Failed to send a leave notice"); - } - } - - if (remove_alias) { - const roomAlias = server.getAliasFromChannel(channel); - try { - yield this.ircBridge.getAppServiceBridge().getIntent().client.deleteAlias(roomAlias); - result.stages.push("Deleted alias for room"); - } - catch (e) { - result.error.push("Failed to remove alias"); - } - } - - // Drop clients from room. - // The provisioner will only drop clients who are not in other rooms. - // It will also leave the MatrixBot. - try { - yield this.ircBridge.getProvisioner()._leaveIfUnprovisioned( - { log: log }, - roomId, - server, - channel - ); - } - catch (e) { - result.error.push("Failed to leave users from room"); - result.error.push(e); - this._wrapJsonResponse(result, false, response); - return; - } - - result.stages.push("Parted clients where applicable."); - this._wrapJsonResponse(result, true, response); -}); - -DebugApi.prototype.inspectUsers = function(regex, response) { - if (!regex) { - this._wrapJsonResponse({ - "error": "'regex' not provided", - }, false, response); - return; - } - try { - const userClients = this.ircBridge.getBridgedClientsForRegex(regex); - const clientsResponse = {}; - Object.keys(userClients).forEach((userId) => { - clientsResponse[userId] = userClients[userId].map((client) => { - if (!client) { - return undefined; - } - return { - channels: client.chanList, - dead: client.isDead(), - server: client.server.domain, - nick: client.nick, - }; - }); - }); - this._wrapJsonResponse({ - users: clientsResponse, - }, true, response); - } - catch (ex) { - this._wrapJsonResponse({ - "error": "Failed to fetch clients for user", - "info": String(ex), - }, false, response); - } -}; - -DebugApi.prototype._wrapJsonReq = function(req, response) { - let body = ""; - req.on("data", function(chunk) { - body += chunk; - }); - return new Promise((resolve, reject) => { - req.on("error", (err) => { - reject(err); - }); - req.on("end", () => { - if (body === "") { - reject({"error": "Body missing"}); - } - try { - body = JSON.parse(body); - resolve(body); - } - catch (err) { - reject(err); - } - }); - }); -} - -DebugApi.prototype._wrapJsonResponse = function(json, isOk, response) { - response.writeHead(isOk === true ? 200 : 500, {"Content-Type": "application/json"}); - response.write(JSON.stringify(json)); - response.end(); -} - -module.exports = DebugApi; diff --git a/src/DebugApi.ts b/src/DebugApi.ts new file mode 100644 index 000000000..cb6469227 --- /dev/null +++ b/src/DebugApi.ts @@ -0,0 +1,451 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import querystring, { ParsedUrlQuery } from "querystring"; +import Bluebird from "bluebird"; +import http, { IncomingMessage, ServerResponse } from "http"; +import { IrcServer } from "./irc/IrcServer"; + +import { BridgeRequest } from "./models/BridgeRequest"; +import { inspect } from "util"; +import { DataStore } from "./datastore/DataStore"; +import { ClientPool } from "./irc/ClientPool"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type IrcBridge = any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type BridgedClient = any; + +export class DebugApi { + constructor(private ircBridge: IrcBridge, private port: number, private servers: IrcServer[], private pool: ClientPool, private token: string) { + + } + + public run () { + log.info("DEBUG API LISTENING ON :%d", this.port); + + http.createServer((req, res) => { + try { + this.onRequest(req, res); + } catch (err) { + if (!res.finished) { + res.end(); + } + log.error(err.stack); + } + }).listen(this.port); + } + + private onRequest(req: IncomingMessage, response: ServerResponse) { + const reqPath = req.url!.split("?"); + const path = reqPath[0]; + const query = querystring.parse(reqPath[1]); + log.debug(req.method + " " + path); + + if (query["access_token"] !== this.token) { + response.writeHead(403, {"Content-Type": "text/plain"}); + response.write("Invalid or missing ?access_token=. " + + "The app service token is required from the registration.\n"); + response.end(); + log.warn("Failed attempt with token " + query["access_token"]); + return; + } + + if (path == "/killUser") { + this.onKillUser(req, response); + return; + } + else if (req.method === "POST" && path == "/reapUsers") { + this.onReapUsers(query, response); + return; + } + else if (req.method === "POST" && path == "/killPortal") { + this.killPortal(req, response); + return; + } + else if (req.method === "GET" && path === "/inspectUsers") { + this.inspectUsers(query["regex"] as string, response); + return; + } + + // Looks like /irc/$domain/user/$user_id + let segs = path.split("/"); + if (segs.length !== 5 || segs[1] !== "irc" || segs[3] !== "user") { + response.writeHead(404, {"Content-Type": "text/plain"}); + response.write("Not a valid debug path.\n"); + response.end(); + return; + } + + let domain = segs[2]; + let user = segs[4]; + + log.debug("Domain: %s User: %s", domain, user); + + const server = this.servers.find((s) => s.domain === domain); + + if (server === undefined) { + response.writeHead(400, {"Content-Type": "text/plain"}); + response.write("Not a valid domain.\n"); + response.end(); + return; + } + + let body = ""; + req.on("data", function(chunk) { + body += chunk; + }); + + req.on("end", () => { + // Create a promise which resolves to a response string + let promise = null; + if (req.method === "GET") { + try { + let resBody = this.getClientState(server, user); + if (!resBody.endsWith("\n")) { + resBody += "\n"; + } + promise = Bluebird.resolve(resBody); + } + catch (err) { + promise = Bluebird.reject(err); + } + } + else if (req.method === "POST") { + promise = this.sendIRCCommand(server, user, body) + } + else { + promise = Bluebird.reject(new Error("Bad HTTP method")); + } + + promise.done((r: string) => { + response.writeHead(200, {"Content-Type": "text/plain"}); + response.write(r); + response.end(); + }, (err: Error) => { + log.error(err.stack); + response.writeHead(500, {"Content-Type": "text/plain"}); + response.write(err + "\n"); + response.end(); + }); + }); + } + + private onKillUser(req: IncomingMessage, response: ServerResponse) { + let bodyStr = ""; + req.on("data", function(chunk) { + bodyStr += chunk; + }); + req.on("end", () => { + let promise = null; + try { + const body = JSON.parse(bodyStr); + if (!body.user_id || !body.reason) { + promise = Promise.reject(new Error("Need user_id and reason")); + } + else { + promise = this.killUser(body.user_id, body.reason); + } + } + catch (err) { + promise = Promise.reject(err); + } + + promise.then((r: string) => { + response.writeHead(200, {"Content-Type": "text/plain"}); + response.write(r + "\n"); + response.end(); + }, (err: Error) => { + log.error(err.stack); + response.writeHead(500, {"Content-Type": "text/plain"}); + response.write(err + "\n"); + response.end(); + }); + }); + } + + public onReapUsers(query: ParsedUrlQuery, response: ServerResponse) { + const msgCb = (msg: string) => { + if (!response.headersSent) { + response.writeHead(200, {"Content-Type": "text/plain"}); + } + response.write(msg + "\n") + }; + this.ircBridge.connectionReap( + msgCb, query["server"], parseInt(query["since"] as string), query["reason"] + ).catch((err: Error) => { + log.error(err.stack); + if (!response.headersSent) { + response.writeHead(500, {"Content-Type": "text/plain"}); + } + response.write(err + "\n"); + }).finally(() => { + response.end(); + }); + } + + private getClient(server: IrcServer, user: string) { + if (!user) { + return this.pool.getBot(server); + } + return this.pool.getBridgedClientByUserId(server, user); + } + + private getClientState(server: IrcServer, user: string) { + log.debug("getClientState(%s,%s)", server.domain, user); + const client = this.getClient(server, user); + if (!client) { + return "User " + user + " does not have a client on " + server.domain; + } + return inspect(client, { colors:true, depth:7 }); + } + + private killUser(userId: string, reason: string) { + const req = new BridgeRequest(this.ircBridge._bridge.getRequestFactory().newRequest()); + const clients = this.pool.getBridgedClientsForUserId(userId); + return this.ircBridge.matrixHandler.quitUser(req, userId, clients, null, reason); + } + + private sendIRCCommand(server: IrcServer, user: string, body: string) { + log.debug("sendIRCCommand(%s,%s,%s)", server.domain, user, body); + const client = this.getClient(server, user); + if (!client) { + return Bluebird.resolve( + "User " + user + " does not have a client on " + server.domain + "\n" + ); + } + if (!client.unsafeClient) { + return Bluebird.resolve( + "There is no underlying client instance.\n" + ); + } + + // store all received response strings + const buffer: string[] = []; + // "raw" can take many forms + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listener = (msg: any) => { + buffer.push(JSON.stringify(msg)); + } + + client.unsafeClient.on("raw", listener); + // turn rn to n so if there are any new lines they are all n. + body = body.replace("\r\n", "\n"); + body.split("\n").forEach((c: string) => { + // IRC protocol require rn + client.unsafeClient.conn.write(c + "\r\n"); + buffer.push(c); + }); + + // wait 3s to pool responses + return Bluebird.delay(3000).then(function() { + // unhook listener to avoid leaking + if (client.unsafeClient) { + client.unsafeClient.removeListener("raw", listener); + } + return buffer.join("\n") + "\n"; + }); + } + + private async killPortal (req: IncomingMessage, response: ServerResponse) { + const store = this.ircBridge.getStore() as DataStore; + const result: { error: string[], stages: string[] } = { + error: [], // string|[string] containing a fatal error or minor errors. + stages: [] // stages completed for removing the room. It's possible it might only + // half complete, and we should make that obvious. + }; + const body = (await this.wrapJsonReq(req, response)) as { + room_id: string; + domain: string; + channel: string; + leave_notice?: boolean; + remove_alias?: boolean; + }; + + if (typeof(body.room_id) !== "string") { + result.error.push(`'room_id' is missing from body or not a string`); + } + + if (typeof(body.domain) !== "string") { + result.error.push(`'domain' is missing from body or not a string`); + } + + if (typeof(body.channel) !== "string") { + result.error.push(`'channel' is missing from body or not a string`); + } + + // Room room_id to lookup and delete the alias from. + const roomId = body["room_id"]; + // IRC server domain + const domain = body["domain"]; + // IRC channel + const channel = body["channel"]; + // Should we tell the room about the deletion. Defaults to true. + const notice = !(body["leave_notice"] === false); + // Should we remove the alias from the room. Defaults to true. + const remove_alias = !(body["remove_alias"] === false); + + if (result.error.length > 0) { + this.wrapJsonResponse(result.error, false, response); + return; + } + + log.warn( + `Requested deletion of portal room alias ${roomId} through debug API + Domain: ${domain} + Channel: ${channel} + Leave Notice: ${notice} + Remove Alias: ${remove_alias}`); + + // Find room + const room = await store.getRoom( + roomId, + domain, + channel, + "alias" + ); + if (room === null) { + result.error.push("Room not found"); + this.wrapJsonResponse(result, false, response); + return; + } + + const server = this.servers.find((srv) => srv.domain === domain); + if (server === undefined) { + result.error.push("Server not found!"); + this.wrapJsonResponse(result, false, response); + return; + } + + // Drop room from room store. + await store.removeRoom( + roomId, + domain, + channel, + "alias" + ); + result.stages.push("Removed room from store"); + + if (notice) { + try { + await this.ircBridge.getAppServiceBridge().getIntent().sendEvent(roomId, "notice", + { + body: `This room has been unbridged from ${channel} (${server.getReadableName()})` + }); + result.stages.push("Left notice in room"); + } + catch (e) { + result.error.push("Failed to send a leave notice"); + } + } + + if (remove_alias) { + const roomAlias = server.getAliasFromChannel(channel); + try { + await this.ircBridge.getAppServiceBridge().getIntent().client.deleteAlias(roomAlias); + result.stages.push("Deleted alias for room"); + } + catch (e) { + result.error.push("Failed to remove alias"); + } + } + + // Drop clients from room. + // The provisioner will only drop clients who are not in other rooms. + // It will also leave the MatrixBot. + try { + await this.ircBridge.getProvisioner()._leaveIfUnprovisioned( + { log: log }, + roomId, + server, + channel + ); + } + catch (e) { + result.error.push("Failed to leave users from room"); + result.error.push(e); + this.wrapJsonResponse(result, false, response); + return; + } + + result.stages.push("Parted clients where applicable."); + this.wrapJsonResponse(result, true, response); + } + + private inspectUsers(regex: string, response: ServerResponse) { + if (!regex) { + this.wrapJsonResponse({ + "error": "'regex' not provided", + }, false, response); + return; + } + try { + const userClients = this.ircBridge.getBridgedClientsForRegex(regex); + const clientsResponse: {[userId: string]: BridgedClient} = {}; + Object.keys(userClients).forEach((userId) => { + clientsResponse[userId] = userClients[userId].map((client: BridgedClient) => { + if (!client) { + return undefined; + } + return { + channels: client.chanList, + dead: client.isDead(), + server: client.server.domain, + nick: client.nick, + }; + }); + }); + this.wrapJsonResponse({ + users: clientsResponse, + }, true, response); + } + catch (ex) { + this.wrapJsonResponse({ + "error": "Failed to fetch clients for user", + "info": String(ex), + }, false, response); + } + }; + + private wrapJsonReq (req: IncomingMessage, response: ServerResponse): Bluebird { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + return new Bluebird((resolve, reject) => { + req.on("error", (err) => { + reject(err); + }); + req.on("end", () => { + if (body === "") { + reject({"error": "Body missing"}); + } + try { + resolve(JSON.parse(body)); + } + catch (err) { + reject(err); + } + }); + }); + } + + private wrapJsonResponse (json: unknown, isOk: boolean, response: ServerResponse) { + response.writeHead(isOk === true ? 200 : 500, {"Content-Type": "application/json"}); + response.write(JSON.stringify(json)); + response.end(); + } +} + +module.exports = DebugApi; From 9fba7d4e11df96a0d9205f3295366e1d148f0fea Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 17:12:40 +0100 Subject: [PATCH 095/350] Newsfile --- changelog.d/829.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/829.misc diff --git a/changelog.d/829.misc b/changelog.d/829.misc new file mode 100644 index 000000000..4b1a9011d --- /dev/null +++ b/changelog.d/829.misc @@ -0,0 +1 @@ +Convert DebugApi to Typescript \ No newline at end of file From c25c28a1a603defb1d70746f27ce72e7e9f44b80 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 17:18:05 +0100 Subject: [PATCH 096/350] Logging fixes after merge --- src/DebugApi.ts | 9 ++++++--- src/irc/ClientPool.ts | 4 ++-- src/logging.ts | 24 +++++++++++++----------- src/models/BridgeRequest.ts | 7 ++++--- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/DebugApi.ts b/src/DebugApi.ts index cb6469227..44f400cc1 100644 --- a/src/DebugApi.ts +++ b/src/DebugApi.ts @@ -23,6 +23,9 @@ import { BridgeRequest } from "./models/BridgeRequest"; import { inspect } from "util"; import { DataStore } from "./datastore/DataStore"; import { ClientPool } from "./irc/ClientPool"; +import { getLogger } from "./logging"; + +const log = getLogger("DebugApi"); // eslint-disable-next-line @typescript-eslint/no-explicit-any type IrcBridge = any; @@ -136,7 +139,7 @@ export class DebugApi { response.write(r); response.end(); }, (err: Error) => { - log.error(err.stack); + log.error(err.stack!); response.writeHead(500, {"Content-Type": "text/plain"}); response.write(err + "\n"); response.end(); @@ -169,7 +172,7 @@ export class DebugApi { response.write(r + "\n"); response.end(); }, (err: Error) => { - log.error(err.stack); + log.error(err.stack!); response.writeHead(500, {"Content-Type": "text/plain"}); response.write(err + "\n"); response.end(); @@ -187,7 +190,7 @@ export class DebugApi { this.ircBridge.connectionReap( msgCb, query["server"], parseInt(query["since"] as string), query["reason"] ).catch((err: Error) => { - log.error(err.stack); + log.error(err.stack!); if (!response.headersSent) { response.writeHead(500, {"Content-Type": "text/plain"}); } diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 03d54a1e8..e5cefecd4 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -15,14 +15,14 @@ limitations under the License. */ import * as stats from "../config/stats"; -import * as logging from "../logging"; +import { getLogger } from "../logging"; import { QueuePool } from "../util/QueuePool"; import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer } from "../irc/IrcServer"; import { AgeCounter, MatrixUser, MatrixRoom } from "matrix-appservice-bridge"; -const log = logging.get("ClientPool"); +const log = getLogger("ClientPool"); // We do not have these yet // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/logging.ts b/src/logging.ts index 581a6bac0..0cefed323 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -57,18 +57,20 @@ const loggers: {[name: string]: LoggerInstance } = { let loggerTransports: TransportInstance[]; // from config +export function timestampFn() { + return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); +}; +export function formatterFn(opts: FormatterFnOpts) { + return opts.timestamp() + ' ' + + opts.level.toUpperCase() + ':' + + (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + + (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + + (opts.meta && opts.meta.dir ? opts.meta.dir : "") + + (undefined !== opts.message ? opts.message : ''); +}; + const makeTransports = function() { - const timestampFn = function() { - return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); - }; - const formatterFn = function(opts: FormatterFnOpts) { - return opts.timestamp() + ' ' + - opts.level.toUpperCase() + ':' + - (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + - (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + - (opts.meta && opts.meta.dir ? opts.meta.dir : "") + - (undefined !== opts.message ? opts.message : ''); - }; + let transports = []; if (loggerConfig.toConsole) { diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index f76b00885..23795c67a 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -import logging = require("../logging"); +import { getLogger, newRequestLogger } from "../logging"; import { Request } from "matrix-appservice-bridge"; -const log = logging.get("req"); +import { LoggerInstance } from "winston"; +const log = getLogger("req"); // We do not have types for logging yet. // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -26,7 +27,7 @@ export class BridgeRequest { private log: Logger; constructor(private req: Request) { const isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; - this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); + this.log = newRequestLogger(log as LoggerInstance, req.getId(), isFromIrc); } getPromise() { From cb90800aed08c72977c5bab86a4e795c4fa8cad9 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 19:10:26 +0100 Subject: [PATCH 097/350] Implement Scheduler in Typescript --- src/irc/Scheduler.js | 69 ---------------------------------- src/irc/Scheduler.ts | 88 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 69 deletions(-) delete mode 100644 src/irc/Scheduler.js create mode 100644 src/irc/Scheduler.ts diff --git a/src/irc/Scheduler.js b/src/irc/Scheduler.js deleted file mode 100644 index 40676130c..000000000 --- a/src/irc/Scheduler.js +++ /dev/null @@ -1,69 +0,0 @@ -/*eslint no-invalid-this: 0*/ -"use strict" -const Promise = require("bluebird"); -const logging = require("../logging"); -var log = logging.get("scheduler"); -const { Queue } = require("../util/Queue.js"); - -/** - * An IRC connection scheduler. Enables ConnectionInstance to reconnect - * in a way that queues reconnection requests and services the FIFO queue at a - * rate determined by ircServer.getReconnectIntervalMs(). - */ - -// Maps domain => Queue -var queues = {}; - -function procFn (item) { - return Promise.delay(item.addedDelayMs).then(item.fn); -} - -function getQueue (server) { - let q = queues[server.domain]; - - if (!q) { - q = new Queue(procFn, server.getReconnectIntervalMs()); - - queues[server.domain] = q; - } - return q; -} - -var Scheduler = { - // Returns a promise that will be resolved when retryConnection returns a promise that - // resolves, in other words, when the connection is made. The promise will reject if the - // promise returned from retryConnection is rejected. - reschedule: Promise.coroutine(function*(server, addedDelayMs, retryConnection, nick) { - var q = getQueue(server); - - var promise = q.enqueue( - `Scheduler.reschedule ${server.domain} ${nick}`, - { - fn: retryConnection, - addedDelayMs: addedDelayMs - } - ); - - log.info( - `Queued scheduled promise for ${server.domain} ${nick}` + - (addedDelayMs > 0 ? ` with ${Math.round(addedDelayMs)}ms added delay`:'') - ); - - log.info( - `Queue for ${server.domain} length = ${q._queue.length}` - ); - - return promise; - }), - - // Reject all queued promises - killAll: function () { - let queueKeys = Object.keys(queues); - for (var i = 0; i < queueKeys.length; i++) { - var q = queues[queueKeys[i]]; - q.killAll(); - } - } -}; - -module.exports = Scheduler; diff --git a/src/irc/Scheduler.ts b/src/irc/Scheduler.ts new file mode 100644 index 000000000..2ae67e5e2 --- /dev/null +++ b/src/irc/Scheduler.ts @@ -0,0 +1,88 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import Bluebird from "bluebird"; +import{ getLogger } from "../logging"; +import { Queue } from "../util/Queue"; +import { IrcServer } from "./IrcServer"; + +const log = getLogger("scheduler"); + +interface QueueItem { + fn: () => Promise, + addedDelayMs: number, +} + +// Maps domain => Queue +const queues: {[domain: string]: Queue} = {}; + +function getQueue (server: IrcServer) { + let q = queues[server.domain]; + + if (!q) { + q = new Queue((item) => { + const { + addedDelayMs, + fn + } = item as QueueItem; + return Bluebird.delay(addedDelayMs).then(fn); + }, server.getReconnectIntervalMs()); + queues[server.domain] = q; + } + return q; +} + +/** + * An IRC connection scheduler. Enables ConnectionInstance to reconnect + * in a way that queues reconnection requests and services the FIFO queue at a + * rate determined by ircServer.getReconnectIntervalMs(). + */ +export default { + // Returns a promise that will be resolved when retryConnection returns a promise that + // resolves, in other words, when the connection is made. The promise will reject if the + // promise returned from retryConnection is rejected. + reschedule: Bluebird.coroutine(function*(server: IrcServer, addedDelayMs: number, retryConnection: () => Promise, nick: string) { + var q = getQueue(server); + + var promise = q.enqueue( + `Scheduler.reschedule ${server.domain} ${nick}`, + { + fn: retryConnection, + addedDelayMs: addedDelayMs + } as QueueItem + ); + + log.info( + `Queued scheduled promise for ${server.domain} ${nick}` + + (addedDelayMs > 0 ? ` with ${Math.round(addedDelayMs)}ms added delay`:'') + ); + + log.info( + `Queue for ${server.domain} length = ${q.size()}` + ); + + return promise; + }), + + // Reject all queued promises + killAll: function () { + let queueKeys = Object.keys(queues); + for (var i = 0; i < queueKeys.length; i++) { + var q = queues[queueKeys[i]]; + q.killAll(); + } + } +} From b6c2e8ee44829d2500a44557065b8523f7f9d0e2 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 19:10:38 +0100 Subject: [PATCH 098/350] Implement ConnectionInstance in Typescript --- src/irc/ConnectionInstance.js | 404 -------------------------------- src/irc/ConnectionInstance.ts | 420 ++++++++++++++++++++++++++++++++++ 2 files changed, 420 insertions(+), 404 deletions(-) delete mode 100644 src/irc/ConnectionInstance.js create mode 100644 src/irc/ConnectionInstance.ts diff --git a/src/irc/ConnectionInstance.js b/src/irc/ConnectionInstance.js deleted file mode 100644 index e28f81a3d..000000000 --- a/src/irc/ConnectionInstance.js +++ /dev/null @@ -1,404 +0,0 @@ -"use strict"; -const irc = require("irc"); -const promiseutil = require("../promiseutil"); -const logging = require("../logging"); -var log = logging.get("client-connection"); -const Scheduler = require("./Scheduler.js"); -const Promise = require("bluebird"); - -// The time we're willing to wait for a connect callback when connecting to IRC. -const CONNECT_TIMEOUT_MS = 30 * 1000; // 30s -// The delay between messages when there are >1 messages to send. -const FLOOD_PROTECTION_DELAY_MS = 700; -// The max amount of time we should wait for the server to ping us before reconnecting. -// Servers ping infrequently (2-3mins) so this should be high enough to allow up -// to 2 pings to lapse before reconnecting (5-6mins). -const PING_TIMEOUT_MS = 1000 * 60 * 10; -// The minimum time to wait between connection attempts if we were disconnected -// due to throttling. -const THROTTLE_WAIT_MS = 20 * 1000; - -// The rate at which to send pings to the IRCd if the client is being quiet for a while. -// Whilst the IRCd *should* be sending pings to us to keep the connection alive, it appears -// that sometimes they don't get around to it and end up ping timing us out. -const PING_RATE_MS = 1000 * 60; - -// String reply of any CTCP Version requests -const CTCP_VERSION = 'matrix-appservice-irc, part of the Matrix.org Network'; - -const CONN_LIMIT_MESSAGES = [ - "too many host connections", // ircd-seven - "no more connections allowed in your connection class", - "this server is full", // unrealircd -] - -// Log an Error object to stderr -function logError(err) { - if (!err || !err.message) { - return; - } - log.error(err.message); -} - -/** - * Create an IRC connection instance. Wraps the node-irc library to handle - * connections correctly. - * @constructor - * @param {IrcClient} ircClient The new IRC client. - * @param {string} domain The domain (for logging purposes) - * @param {string} nick The nick (for logging purposes) - */ -function ConnectionInstance(ircClient, domain, nick) { - this.client = ircClient; - this.domain = domain; - this.nick = nick; - this._listenForErrors(); - this._listenForPings(); - this._listenForCTCPVersions(); - this.dead = false; - this.state = "created"; // created|connecting|connected - this._connectDefer = promiseutil.defer(); - this._pingRateTimerId = null; - this._clientSidePingTimeoutTimerId = null; -} - -/** - * Connect this client to the server. There are zero guarantees this will ever - * connect. - * @return {Promise} Resolves if connected; rejects if failed to connect. - */ -ConnectionInstance.prototype.connect = function() { - if (this.dead) { - throw new Error("connect() called on dead client: " + this.nick); - } - this.state = "connecting"; - var self = this; - var domain = self.domain; - var gotConnectedCallback = false; - setTimeout(function() { - if (!gotConnectedCallback && !self.dead) { - log.error( - "%s@%s still not connected after %sms. Killing connection.", - self.nick, domain, CONNECT_TIMEOUT_MS - ); - self.disconnect("timeout").catch(logError); - } - }, CONNECT_TIMEOUT_MS); - - self.client.connect(1, function() { - gotConnectedCallback = true; - self.state = "connected"; - self._resetPingSendTimer(); - self._connectDefer.resolve(self); - }); - return this._connectDefer.promise; -}; - -/** - * Blow away the connection. You MUST destroy this object afterwards. - * @param {string} reason - Reason to reject with. One of: - * throttled|irc_error|net_error|timeout|raw_error|toomanyconns - */ -ConnectionInstance.prototype.disconnect = function(reason) { - if (this.dead) { - return Promise.resolve(); - } - log.info( - "disconnect()ing %s@%s - %s", this.nick, this.domain, reason - ); - this.dead = true; - - return new Promise((resolve, reject) => { - // close the connection - this.client.disconnect(reason, function() {}); - // remove timers - if (this._pingRateTimerId) { - clearTimeout(this._pingRateTimerId); - this._pingRateTimerId = null; - } - if (this._clientSidePingTimeoutTimerId) { - clearTimeout(this._clientSidePingTimeoutTimerId); - this._clientSidePingTimeoutTimerId = null; - } - if (this.state !== "connected") { - // we never resolved this defer, so reject it. - this._connectDefer.reject(new Error(reason)); - } - if (this.state === "connected" && this.onDisconnect) { - // we only invoke onDisconnect once we've had a successful connect. - // Connection *attempts* are managed by the create() function so if we - // call this now it would potentially invoke this 3 times (once per - // connection instance!). Each time would have dead=false as they are - // separate objects. - this.onDisconnect(reason); - } - resolve(); - }); -}; - -ConnectionInstance.prototype.addListener = function(eventName, fn) { - var self = this; - this.client.addListener(eventName, function() { - if (self.dead) { - log.error( - "%s@%s RECV a %s event for a dead connection", - self.nick, self.domain, eventName - ); - return; - } - // do the callback - fn.apply(fn, arguments); - }); -}; - -ConnectionInstance.prototype._listenForErrors = function() { - var self = this; - var domain = self.domain; - var nick = self.nick; - self.client.addListener("error", function(err) { - log.error("Server: %s (%s) Error: %s", domain, nick, JSON.stringify(err)); - // We should disconnect the client for some but not all error codes. This - // list is a list of codes which we will NOT disconnect the client for. - var failCodes = [ - "err_nosuchchannel", "err_toomanychannels", "err_channelisfull", - "err_inviteonlychan", "err_bannedfromchan", "err_badchannelkey", - "err_needreggednick", "err_nosuchnick", "err_cannotsendtochan", - "err_toomanychannels", "err_erroneusnickname", "err_usernotinchannel", - "err_notonchannel", "err_useronchannel", "err_notregistered", - "err_alreadyregistred", "err_noprivileges", "err_chanoprivsneeded", - "err_banonchan", "err_nickcollision", "err_nicknameinuse", - "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", - "err_nicktoofast", "err_unknowncommand", "err_unavailresource", - "err_umodeunknownflag", "err_nononreg" - ]; - if (err && err.command) { - if (failCodes.indexOf(err.command) !== -1) { - return; // don't disconnect for these error codes. - } - } - if (err && err.command === "err_yourebannedcreep") { - self.disconnect("banned").catch(logError); - return; - } - self.disconnect("irc_error").catch(logError); - }); - self.client.addListener("netError", function(err) { - log.error( - "Server: %s (%s) Network Error: %s", domain, nick, - JSON.stringify(err, undefined, 2) - ); - self.disconnect("net_error").catch(logError); - }); - self.client.addListener("abort", function() { - log.error( - "Server: %s (%s) Connection Aborted", domain, nick - ); - self.disconnect("net_error").catch(logError); - }); - self.client.addListener("raw", function(msg) { - if (logging.isVerbose()) { - log.debug( - "%s@%s: %s", nick, domain, JSON.stringify(msg) - ); - } - if (msg && (msg.command === "ERROR" || msg.rawCommand === "ERROR")) { - log.error( - "%s@%s: %s", nick, domain, JSON.stringify(msg) - ); - var wasThrottled = false; - if (msg.args) { - var errText = ("" + msg.args[0]) || ""; - errText = errText.toLowerCase(); - wasThrottled = errText.indexOf("throttl") !== -1; - if (wasThrottled) { - self.disconnect("throttled").catch(logError); - return; - } - const wasBanned = errText.includes("banned") || errText.includes("k-lined"); - if (wasBanned) { - self.disconnect("banned").catch(logError); - return; - } - const tooManyHosts = CONN_LIMIT_MESSAGES.find((connLimitMsg) => { - return errText.includes(connLimitMsg); - }) !== undefined; - if (tooManyHosts) { - self.disconnect("toomanyconns").catch(logError); - return; - } - } - if (!wasThrottled) { - self.disconnect("raw_error").catch(logError); - } - } - }); -}; - -ConnectionInstance.prototype._listenForPings = function() { - // BOTS-65 : A client can get ping timed out and not reconnect. - // ------------------------------------------------------------ - // The client is doing IRC ping/pongs, but there is no check to say - // "hey, the server hasn't pinged me in a while, it's probably dead". The - // RFC for pings states that pings are sent "if no other activity detected - // from a connection." so we need to count anything we shove down the wire - // as a ping refresh. - var self = this; - var domain = self.domain; - var nick = self.nick; - function _keepAlivePing() { // refresh the ping timer - if (self._clientSidePingTimeoutTimerId) { - clearTimeout(self._clientSidePingTimeoutTimerId); - } - self._clientSidePingTimeoutTimerId = setTimeout(function() { - log.info( - "Ping timeout: knifing connection for %s on %s", - domain, nick - ); - // Just emit an netError which clients need to handle anyway. - self.client.emit("netError", { - msg: "Client-side ping timeout" - }); - }, PING_TIMEOUT_MS); - } - self.client.on("ping", function(svr) { - log.debug("Received ping from %s directed at %s", svr, nick); - _keepAlivePing(); - }); - // decorate client.send to refresh the timer - var realSend = self.client.send; - self.client.send = function(command) { - _keepAlivePing(); - self._resetPingSendTimer(); // sending a message counts as a ping - realSend.apply(self.client, arguments); - }; -}; - -ConnectionInstance.prototype._listenForCTCPVersions = function() { - const self = this; - self.client.addListener("ctcp-version", function (from) { - self.client.ctcp(from, 'reply', `VERSION ${CTCP_VERSION}`); - }); -}; - -ConnectionInstance.prototype._resetPingSendTimer = function() { - // reset the ping rate timer - if (this._pingRateTimerId) { - clearTimeout(this._pingRateTimerId); - } - this._pingRateTimerId = setTimeout(() => { - if (this.dead) { - return; - } - // Do what XChat does - this.client.send("PING", "LAG" + Date.now()); - // keep doing it. - this._resetPingSendTimer(); - }, PING_RATE_MS); -}; - -/** - * Create an IRC client connection and connect to it. - * @param {IrcServer} server The server to connect to. - * @param {Object} opts Options for this connection. - * @param {string} opts.nick The nick to use. - * @param {string} opts.username The username to use. - * @param {string} opts.realname The real name of the user. - * @param {string} opts.password The password to give NickServ. - * @param {string} opts.localAddress The local address to bind to when connecting. - * @param {Function} onCreatedCallback Called with the client when created. - * @return {Promise} Resolves to an ConnectionInstance or rejects. - */ -ConnectionInstance.create = Promise.coroutine(function*(server, opts, onCreatedCallback) { - if (!opts.nick || !server) { - throw new Error("Bad inputs. Nick: " + opts.nick); - } - onCreatedCallback = onCreatedCallback || function() {}; - let connectionOpts = { - userName: opts.username, - realName: opts.realname, - password: opts.password, - localAddress: opts.localAddress, - autoConnect: false, - autoRejoin: false, - floodProtection: true, - floodProtectionDelay: FLOOD_PROTECTION_DELAY_MS, - port: server.getPort(), - selfSigned: server.useSslSelfSigned(), - certExpired: server.allowExpiredCerts(), - retryCount: 0, - family: server.getIpv6Prefix() || server.getIpv6Only() ? 6 : null, - bustRfc3484: true, - sasl: opts.password ? server.useSasl() : false, - }; - - if (server.useSsl()) { - connectionOpts.secure = { ca: server.getCA() }; - } - - // Returns: A promise which resolves to a ConnectionInstance - let retryConnection = () => { - let nodeClient = new irc.Client( - server.randomDomain(), opts.nick, connectionOpts - ); - let inst = new ConnectionInstance( - nodeClient, server.domain, opts.nick - ); - onCreatedCallback(inst); - return inst.connect(); - }; - - let connAttempts = 0; - let retryTimeMs = 0; - const BASE_RETRY_TIME_MS = 1000; - while (true) { - try { - if (server.getReconnectIntervalMs() > 0) { - // wait until scheduled - let cli = yield Scheduler.reschedule( - server, retryTimeMs, retryConnection, opts.nick - ); - return cli; - } - // Try to connect immediately: we'll wait if we fail. - let cli = yield retryConnection(); - return cli; - } - catch (err) { - connAttempts += 1; - log.error( - `ConnectionInstance.connect failed after ${connAttempts} attempts (${err.message})` - ); - - if (err.message === "throttled") { - retryTimeMs += THROTTLE_WAIT_MS; - } - - if (err.message === "banned") { - log.error( - `${opts.nick} is banned from ${server.domain}, ` + - `throwing` - ); - throw new Error("User is banned from the network."); - // If the user is banned, we should part them from any rooms. - } - - if (err.message === "toomanyconns") { - log.error( - `User ${opts.nick} was ILINED. This may be the network limiting us!` - ); - throw new Error("Connection was ILINED. We cannot retry this."); - } - - // always set a staggered delay here to avoid thundering herd - // problems on mass-disconnects - let delay = (BASE_RETRY_TIME_MS * Math.random())+ retryTimeMs + - Math.round((connAttempts * 1000) * Math.random()); - log.info(`Retrying connection for ${opts.nick} on ${server.domain} `+ - `in ${delay}ms (attempts ${connAttempts})`); - yield Promise.delay(delay); - } - } -}); - - -module.exports = ConnectionInstance; diff --git a/src/irc/ConnectionInstance.ts b/src/irc/ConnectionInstance.ts new file mode 100644 index 000000000..9219ef464 --- /dev/null +++ b/src/irc/ConnectionInstance.ts @@ -0,0 +1,420 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const irc = require("irc"); + +import * as promiseutil from "../promiseutil"; +import Scheduler from "./Scheduler"; +import * as logging from "../logging"; +import Bluebird from "bluebird"; +import { Defer } from "../promiseutil"; +import { IrcServer } from "./IrcServer"; + +const log = logging.get("client-connection"); + + +// The time we're willing to wait for a connect callback when connecting to IRC. +const CONNECT_TIMEOUT_MS = 30 * 1000; // 30s +// The delay between messages when there are >1 messages to send. +const FLOOD_PROTECTION_DELAY_MS = 700; +// The max amount of time we should wait for the server to ping us before reconnecting. +// Servers ping infrequently (2-3mins) so this should be high enough to allow up +// to 2 pings to lapse before reconnecting (5-6mins). +const PING_TIMEOUT_MS = 1000 * 60 * 10; +// The minimum time to wait between connection attempts if we were disconnected +// due to throttling. +const THROTTLE_WAIT_MS = 20 * 1000; + +// The rate at which to send pings to the IRCd if the client is being quiet for a while. +// Whilst the IRCd *should* be sending pings to us to keep the connection alive, it appears +// that sometimes they don't get around to it and end up ping timing us out. +const PING_RATE_MS = 1000 * 60; + +// String reply of any CTCP Version requests +const CTCP_VERSION = 'matrix-appservice-irc, part of the Matrix.org Network'; + +const CONN_LIMIT_MESSAGES = [ + "too many host connections", // ircd-seven + "no more connections allowed in your connection class", + "this server is full", // unrealircd +]; + +// Log an Error object to stderr +function logError(err: Error) { + if (!err || !err.message) { + return; + } + log.error(err.message); +} + +export interface ConnectionOpts { + localAddress: string; + password?: string; + realname: string; + username: string; + nick: string; + secure?: { + ca?: string; + } +} + +export class ConnectionInstance { + private dead: boolean = false; + private state: "created"|"connecting"|"connected" = "created"; + private pingRateTimerId: NodeJS.Timer|null = null; + private clientSidePingTimeoutTimerId: NodeJS.Timer|null = null; + private connectDefer: Defer; + public onDisconnect?: (reason: string) => void; + /** + * Create an IRC connection instance. Wraps the node-irc library to handle + * connections correctly. + * @constructor + * @param {IrcClient} ircClient The new IRC client. + * @param {string} domain The domain (for logging purposes) + * @param {string} nick The nick (for logging purposes) + */ + constructor (private readonly client: any, private readonly domain: string, private nick: string) { + this.listenForErrors(); + this.listenForPings(); + this.listenForCTCPVersions(); + this.connectDefer = promiseutil.defer(); + } + + /** + * Connect this client to the server. There are zero guarantees this will ever + * connect. + * @return {Promise} Resolves if connected; rejects if failed to connect. + */ + public connect(): Promise { + if (this.dead) { + throw new Error("connect() called on dead client: " + this.nick); + } + this.state = "connecting"; + let gotConnectedCallback = false; + setTimeout(() => { + if (!gotConnectedCallback && !this.dead) { + log.error( + "%s@%s still not connected after %sms. Killing connection.", + this.nick, this.domain, CONNECT_TIMEOUT_MS + ); + this.disconnect("timeout").catch(logError); + } + }, CONNECT_TIMEOUT_MS); + + this.client.connect(1, () => { + gotConnectedCallback = true; + this.state = "connected"; + this.resetPingSendTimer(); + this.connectDefer.resolve(this); + }); + return this.connectDefer.promise; + } + + /** + * Blow away the connection. You MUST destroy this object afterwards. + * @param {string} reason - Reason to reject with. One of: + * throttled|irc_error|net_error|timeout|raw_error|toomanyconns|banned + */ + public disconnect(reason: "throttled"|"irc_error"|"net_error"|"timeout"|"raw_error"|"toomanyconns"|"banned") { + if (this.dead) { + return Bluebird.resolve(); + } + log.info( + "disconnect()ing %s@%s - %s", this.nick, this.domain, reason + ); + this.dead = true; + + return new Bluebird((resolve) => { + // close the connection + this.client.disconnect(reason, () => {}); + // remove timers + if (this.pingRateTimerId) { + clearTimeout(this.pingRateTimerId); + this.pingRateTimerId = null; + } + if (this.clientSidePingTimeoutTimerId) { + clearTimeout(this.clientSidePingTimeoutTimerId); + this.clientSidePingTimeoutTimerId = null; + } + if (this.state !== "connected") { + // we never resolved this defer, so reject it. + this.connectDefer.reject(new Error(reason)); + } + if (this.state === "connected" && this.onDisconnect) { + // we only invoke onDisconnect once we've had a successful connect. + // Connection *attempts* are managed by the create() function so if we + // call this now it would potentially invoke this 3 times (once per + // connection instance!). Each time would have dead=false as they are + // separate objects. + this.onDisconnect(reason); + } + resolve(); + }); + } + + public addListener(eventName: string, fn: (item: IArguments) => void) { + this.client.addListener(eventName, () => { + if (this.dead) { + log.error( + "%s@%s RECV a %s event for a dead connection", + this.nick, this.domain, eventName + ); + return; + } + // do the callback + fn.apply(fn, arguments as any); + }); + } + + private listenForErrors() { + this.client.addListener("error", (err?: {command?: string}) => { + log.error("Server: %s (%s) Error: %s", this.domain, this.nick, JSON.stringify(err)); + // We should disconnect the client for some but not all error codes. This + // list is a list of codes which we will NOT disconnect the client for. + const failCodes = [ + "err_nosuchchannel", "err_toomanychannels", "err_channelisfull", + "err_inviteonlychan", "err_bannedfromchan", "err_badchannelkey", + "err_needreggednick", "err_nosuchnick", "err_cannotsendtochan", + "err_toomanychannels", "err_erroneusnickname", "err_usernotinchannel", + "err_notonchannel", "err_useronchannel", "err_notregistered", + "err_alreadyregistred", "err_noprivileges", "err_chanoprivsneeded", + "err_banonchan", "err_nickcollision", "err_nicknameinuse", + "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", + "err_nicktoofast", "err_unknowncommand", "err_unavailresource", + "err_umodeunknownflag", "err_nononreg" + ]; + if (err && err.command) { + if (failCodes.indexOf(err.command) !== -1) { + return; // don't disconnect for these error codes. + } + } + if (err && err.command === "err_yourebannedcreep") { + this.disconnect("banned").catch(logError); + return; + } + this.disconnect("irc_error").catch(logError); + }); + this.client.addListener("netError", (err: unknown) => { + log.error( + "Server: %s (%s) Network Error: %s", this.domain, this.nick, + JSON.stringify(err, undefined, 2) + ); + this.disconnect("net_error").catch(logError); + }); + this.client.addListener("abort", () => { + log.error( + "Server: %s (%s) Connection Aborted", this.domain, this.nick + ); + this.disconnect("net_error").catch(logError); + }); + this.client.addListener("raw", (msg?: {command?: string, rawCommand: string, args?: string[]}) => { + if (logging.isVerbose()) { + log.debug( + "%s@%s: %s", this.nick, this.domain, JSON.stringify(msg) + ); + } + if (msg && (msg.command === "ERROR" || msg.rawCommand === "ERROR")) { + log.error( + "%s@%s: %s", this.nick, this.domain, JSON.stringify(msg) + ); + var wasThrottled = false; + if (msg.args) { + var errText = ("" + msg.args[0]) || ""; + errText = errText.toLowerCase(); + wasThrottled = errText.indexOf("throttl") !== -1; + if (wasThrottled) { + this.disconnect("throttled").catch(logError); + return; + } + const wasBanned = errText.includes("banned") || errText.includes("k-lined"); + if (wasBanned) { + this.disconnect("banned").catch(logError); + return; + } + const tooManyHosts = CONN_LIMIT_MESSAGES.find((connLimitMsg) => { + return errText.includes(connLimitMsg); + }) !== undefined; + if (tooManyHosts) { + this.disconnect("toomanyconns").catch(logError); + return; + } + } + if (!wasThrottled) { + this.disconnect("raw_error").catch(logError); + } + } + }); + }; + + private listenForPings() { + // BOTS-65 : A client can get ping timed out and not reconnect. + // ------------------------------------------------------------ + // The client is doing IRC ping/pongs, but there is no check to say + // "hey, the server hasn't pinged me in a while, it's probably dead". The + // RFC for pings states that pings are sent "if no other activity detected + // from a connection." so we need to count anything we shove down the wire + // as a ping refresh. + const keepAlivePing = () => { // refresh the ping timer + if (this.clientSidePingTimeoutTimerId) { + clearTimeout(this.clientSidePingTimeoutTimerId); + } + this.clientSidePingTimeoutTimerId = setTimeout(() => { + log.info( + "Ping timeout: knifing connection for %s on %s", + this.domain, this.nick, + ); + // Just emit an netError which clients need to handle anyway. + this.client.emit("netError", { + msg: "Client-side ping timeout" + }); + }, PING_TIMEOUT_MS); + } + this.client.on("ping", (svr: string) => { + log.debug("Received ping from %s directed at %s", svr, this.nick); + keepAlivePing(); + }); + // decorate client.send to refresh the timer + const realSend = this.client.send; + this.client.send = (command: string) => { + keepAlivePing(); + this.resetPingSendTimer(); // sending a message counts as a ping + realSend.apply(this.client, arguments); + }; + }; + + private listenForCTCPVersions() { + this.client.addListener("ctcp-version", (from: string) => { + this.client.ctcp(from, 'reply', `VERSION ${CTCP_VERSION}`); + }); + }; + + private resetPingSendTimer() { + // reset the ping rate timer + if (this.pingRateTimerId) { + clearTimeout(this.pingRateTimerId); + } + this.pingRateTimerId = setTimeout(() => { + if (this.dead) { + return; + } + // Do what XChat does + this.client.send("PING", "LAG" + Date.now()); + // keep doing it. + this.resetPingSendTimer(); + }, PING_RATE_MS); + } + + /** + * Create an IRC client connection and connect to it. + * @param {IrcServer} server The server to connect to. + * @param {Object} opts Options for this connection. + * @param {string} opts.nick The nick to use. + * @param {string} opts.username The username to use. + * @param {string} opts.realname The real name of the user. + * @param {string} opts.password The password to give NickServ. + * @param {string} opts.localAddress The local address to bind to when connecting. + * @param {Function} onCreatedCallback Called with the client when created. + * @return {Promise} Resolves to an ConnectionInstance or rejects. + */ + public static async create (server: IrcServer, opts: ConnectionOpts, onCreatedCallback: (inst: ConnectionInstance) => void) { + if (!opts.nick || !server) { + throw new Error("Bad inputs. Nick: " + opts.nick); + } + onCreatedCallback = onCreatedCallback || function() {}; + const connectionOpts = { + userName: opts.username, + realName: opts.realname, + password: opts.password, + localAddress: opts.localAddress, + autoConnect: false, + autoRejoin: false, + floodProtection: true, + floodProtectionDelay: FLOOD_PROTECTION_DELAY_MS, + port: server.getPort(), + selfSigned: server.useSslSelfSigned(), + certExpired: server.allowExpiredCerts(), + retryCount: 0, + family: server.getIpv6Prefix() || server.getIpv6Only() ? 6 : null, + bustRfc3484: true, + sasl: opts.password ? server.useSasl() : false, + secure: server.useSsl() ? { ca: server.getCA() } : undefined, + }; + + // Returns: A promise which resolves to a ConnectionInstance + let retryConnection = () => { + let nodeClient = new irc.Client( + server.randomDomain(), opts.nick, connectionOpts + ); + let inst = new ConnectionInstance( + nodeClient, server.domain, opts.nick + ); + onCreatedCallback(inst); + return inst.connect(); + }; + + let connAttempts = 0; + let retryTimeMs = 0; + const BASE_RETRY_TIME_MS = 1000; + while (true) { + try { + if (server.getReconnectIntervalMs() > 0) { + // wait until scheduled + let cli = await Scheduler.reschedule( + server, retryTimeMs, retryConnection, opts.nick + ); + return cli; + } + // Try to connect immediately: we'll wait if we fail. + let cli = await retryConnection(); + return cli; + } + catch (err) { + connAttempts += 1; + log.error( + `ConnectionInstance.connect failed after ${connAttempts} attempts (${err.message})` + ); + + if (err.message === "throttled") { + retryTimeMs += THROTTLE_WAIT_MS; + } + + if (err.message === "banned") { + log.error( + `${opts.nick} is banned from ${server.domain}, ` + + `throwing` + ); + throw new Error("User is banned from the network."); + // If the user is banned, we should part them from any rooms. + } + + if (err.message === "toomanyconns") { + log.error( + `User ${opts.nick} was ILINED. This may be the network limiting us!` + ); + throw new Error("Connection was ILINED. We cannot retry this."); + } + + // always set a staggered delay here to avoid thundering herd + // problems on mass-disconnects + const delay = (BASE_RETRY_TIME_MS * Math.random())+ retryTimeMs + + Math.round((connAttempts * 1000) * Math.random()); + log.info(`Retrying connection for ${opts.nick} on ${server.domain} `+ + `in ${delay}ms (attempts ${connAttempts})`); + await Bluebird.delay(delay); + } + } + } +} \ No newline at end of file From cfd8a7fdb0a74a373ce0e59e1caa1a913522d5a3 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 19:10:50 +0100 Subject: [PATCH 099/350] getLogger --- src/datastore/NedbDataStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 4117666fd..a3884259a 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -17,7 +17,7 @@ limitations under the License. import Bluebird from "bluebird"; import { IrcRoom } from "../models/IrcRoom"; import { IrcClientConfig, IrcClientConfigSeralized } from "../models/IrcClientConfig" -import * as logging from "../logging"; +import { getLogger } from "../logging"; import { MatrixRoom, MatrixUser, RemoteUser, RemoteRoom, UserBridgeStore, RoomBridgeStore, Entry } from "matrix-appservice-bridge"; @@ -25,7 +25,7 @@ import { DataStore, RoomOrigin, ChannelMappings, UserFeatures } from "./DataStor import { IrcServer, IrcServerConfig } from "../irc/IrcServer"; import { StringCrypto } from "./StringCrypto"; -const log = logging.get("NeDBDataStore"); +const log = getLogger("NeDBDataStore"); interface ClientConfigMap { [domain: string]: IrcClientConfigSeralized; From 38e26b9b0cd7ae7cabc3b88a5c712f2a9606d75c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 19:40:07 +0100 Subject: [PATCH 100/350] [WIP] BridgedClient partial conversion --- src/irc/BridgedClient.ts | 791 ++++++++++++++++++++++++++++++++++ src/irc/ConnectionInstance.ts | 6 +- src/irc/Ident.ts | 6 +- src/irc/IrcServer.ts | 8 +- 4 files changed, 802 insertions(+), 9 deletions(-) create mode 100644 src/irc/BridgedClient.ts diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts new file mode 100644 index 000000000..dc3248d35 --- /dev/null +++ b/src/irc/BridgedClient.ts @@ -0,0 +1,791 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import Bluebird from "bluebird"; +import * as promiseutil from "../promiseutil"; +import util from "util"; +import { EventEmitter } from "events"; +import * as Ident from "./Ident" +import { ConnectionInstance, InstanceDisconnectReason } from "./ConnectionInstance"; +import { IrcRoom } from "../models/IrcRoom"; +import { getLogger } from "../logging"; +import { IrcServer } from "./IrcServer"; +import { IrcClientConfig } from "../models/IrcClientConfig"; +import { MatrixUser } from "matrix-appservice-bridge"; +import { LoggerInstance } from "winston"; + +const log = getLogger("BridgedClient"); + +// The length of time to wait before trying to join the channel again +const JOIN_TIMEOUT_MS = 15 * 1000; // 15s +const NICK_DELAY_TIMER_MS = 10 * 1000; // 10s + +type EventBroker = any; +type IdentGenerator = any; +type Ipv6Generator = any; +type IrcClient = any; + +export class BridgedClient extends EventEmitter { + public readonly userId: string|null; + public readonly displayName: string|null; + private _nick: string; + private readonly id: string; + private readonly password?: string; + private unsafeClient: IrcClient|null; + private lastActionTs: number; + private inst: ConnectionInstance|null; + private instCreationFailed: boolean; + private explicitDisconnect: boolean; + private disconnectReason: string|null; + private chanList: string[]; + private connectDefer: promiseutil.Defer; + private log: LoggerInstance; + private cachedOperatorNicksInfo: {[channel: string]: any}; + /** + * Create a new bridged IRC client. + * @constructor + * @param {IrcServer} server + * @param {IrcClientConfig} ircClientConfig : The IRC user to create a connection for. + * @param {MatrixUser} matrixUser : Optional. The matrix user representing this virtual IRC user. + * @param {boolean} isBot : True if this is the bot + * @param {IrcEventBroker} eventBroker + * @param {IdentGenerator} identGenerator + * @param {Ipv6Generator} ipv6Generator + */ + constructor( + public readonly server: IrcServer, + private clientConfig: IrcClientConfig, + public readonly matrixUser: MatrixUser|undefined, + public readonly isBot: boolean, + private readonly eventBroker: EventBroker, + private readonly identGenerator: IdentGenerator, + private readonly ipv6Generator: Ipv6Generator) { + super(); + this.userId = matrixUser ? matrixUser.getId() : null; + this.displayName = matrixUser ? matrixUser.getDisplayName() : null; + this._nick = this.getValidNick( + clientConfig.getDesiredNick() || server.getNick(this.userId!, this.displayName || undefined), + false + ); + this.password = ( + clientConfig.getPassword() ? clientConfig.getPassword() : server.config.password + ); + + this.lastActionTs = Date.now(); + this.inst = null; + this.instCreationFailed = false; + this.explicitDisconnect = false; + this.disconnectReason = null; + this.chanList = []; + this.connectDefer = promiseutil.defer(); + this.id = (Math.random() * 1e20).toString(36); + this.unsafeClient = null; + // decorate log lines with the nick and domain, along with an instance id + var prefix = "<" + this.nick + "@" + this.server.domain + "#" + this.id + "> "; + if (this.userId) { + prefix += "(" + this.userId + ") "; + } + this.log = { + debug: function() { + arguments[0] = prefix + arguments[0]; + log.debug.apply(log, arguments as any); + }, + info: function() { + arguments[0] = prefix + arguments[0]; + log.info.apply(log, arguments as any); + }, + error: function() { + arguments[0] = prefix + arguments[0]; + log.error.apply(log, arguments as any); + } + } as unknown as LoggerInstance; + + this.cachedOperatorNicksInfo = { + // $channel : info + } + } + + public get nick() : string { + return this._nick; + } + + + public getClientConfig() { + return this.clientConfig; + } + + public kill(reason: string) { + // Nullify so that no further commands can be issued + // via unsafeClient, which should be null checked + // anyway as it is not instantiated until a connection + // has occurred. + this.unsafeClient = null; + // kill connection instance + log.info('Killing client ', this.nick); + return this.disconnect(reason || "Bridged client killed"); + } + + public isDead() { + if (this.instCreationFailed || (this.inst && this.inst.dead)) { + return true; + } + return false; + } + + public toString() { + let domain = this.server ? this.server.domain : "NO_DOMAIN"; + return `${this.nick}@${domain}#${this.id}~${this.userId}`; + } + + /** + * @return {ConnectionInstance} A new connected connection instance. + */ + public async connect(): Promise { + try { + let nameInfo = yield this._identGenerator.getIrcNames( + this._clientConfig, this.matrixUser + ); + if (this.server.getIpv6Prefix()) { + // side-effects setting the IPv6 address on the client config + yield this._ipv6Generator.generate( + this.server.getIpv6Prefix(), this._clientConfig + ); + } + this.log.info( + "Connecting to IRC server %s as %s (user=%s)", + this.server.domain, this.nick, nameInfo.username + ); + this.eventBroker.sendMetadata(this, + `Connecting to the IRC network '${this.server.domain}' as ${this.nick}...` + ); + + const connInst = yield ConnectionInstance.create(server, { + nick: this.nick, + username: nameInfo.username, + realname: nameInfo.realname, + password: this.password, + // Don't use stored IPv6 addresses unless they have a prefix else they + // won't be able to turn off IPv6! + localAddress: ( + this.server.getIpv6Prefix() ? this._clientConfig.getIpv6Address() : undefined + ) + }, (inst) => { + this._onConnectionCreated(inst, nameInfo); + }); + + this.inst = connInst; + this.unsafeClient = connInst.client; + this.emit("client-connected", this); + // we may have been assigned a different nick, so update it from source + this._nick = connInst.client.nick; + this.connectDefer.resolve(); + this.keepAlive(); + + let connectText = ( + `You've been connected to the IRC network '${this.server.domain}' as ${this.nick}.` + ); + + let userModes = this.server.getUserModes(); + if (userModes.length > 0 && !this.isBot) { + // These can fail, but the generic error listener will catch them and send them + // into the same room as the connect text, so it's probably good enough to not + // explicitly handle them. + this.unsafeClient.setUserMode("+" + userModes); + connectText += ( + ` User modes +${userModes} have been set.` + ); + } + + this.eventBroker.sendMetadata(this, connectText); + + connInst.client.addListener("nick", (old, newNick) => { + if (old === this.nick) { + this.log.info( + "NICK: Nick changed from '" + old + "' to '" + newNick + "'." + ); + this._nick = newNick; + this.emit("nick-change", this, old, newNick); + } + }); + connInst.client.addListener("error", (err) => { + // Errors we MUST notify the user about, regardless of the bridge's admin room config. + const ERRORS_TO_FORCE = ["err_nononreg"] + if (!err || !err.command || connInst.dead) { + return; + } + var msg = "Received an error on " + this.server.domain + ": " + err.command + "\n"; + msg += JSON.stringify(err.args); + this.eventBroker.sendMetadata(this, msg, ERRORS_TO_FORCE.includes(err.command)); + }); + return connInst; + } + catch (err) { + this.log.debug("Failed to connect."); + this.instCreationFailed = true; + throw err; + } + }); + + public disconnect(reason: InstanceDisconnectReason) { + this.explicitDisconnect = true; + if (!this.inst || this.inst.dead) { + return Promise.resolve(); + } + return this.inst.disconnect(reason); + } + + /** + * Change this user's nick. + * @param {string} newNick The new nick for the user. + * @param {boolean} throwOnInvalid True to throw an error on invalid nicks + * instead of coercing them. + * @return {Promise} Which resolves to a message to be sent to the user. + */ + public changeNick(newNick: string, throwOnInvalid: boolean): Promise { + let validNick = newNick; + try { + validNick = this.getValidNick(newNick, throwOnInvalid); + if (validNick === this.nick) { + return Promise.resolve(`Your nick is already '${validNick}'.`); + } + } + catch (err) { + return Promise.reject(err); + } + if (!this.unsafeClient) { + return Promise.reject(new Error("You are not connected to the network.")); + } + + return new Promise((resolve, reject) => { + let nickListener, nickErrListener; + const timeoutId = setTimeout(() => { + this.log.error("Timed out trying to change nick to %s", validNick); + // may have d/ced between sending nick change and now so recheck + if (this.unsafeClient) { + this.unsafeClient.removeListener("nick", nickListener); + this.unsafeClient.removeListener("error", nickErrListener); + } + reject(new Error("Timed out waiting for a response to change nick.")); + }, NICK_DELAY_TIMER_MS); + nickListener = (old, n) => { + clearTimeout(timeoutId); + this.unsafeClient.removeListener("error", nickErrListener); + resolve("Nick changed from '" + old + "' to '" + n + "'."); + } + nickErrListener = (err) => { + if (!err || !err.command) { return; } + var failCodes = [ + "err_banonchan", "err_nickcollision", "err_nicknameinuse", + "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", + "err_nicktoofast", "err_unavailresource" + ]; + if (failCodes.indexOf(err.command) !== -1) { + this.log.error("Nick change error : %s", err.command); + clearTimeout(timeoutId); + this.unsafeClient.removeListener("nick", nickListener); + reject(new Error("Failed to change nick: " + err.command)); + } + } + this.unsafeClient.once("nick", nickListener); + this.unsafeClient.once("error", nickErrListener); + this.unsafeClient.send("NICK", validNick); + }); + } + + + public leaveChannel(channel: string, reason: string) { + reason = reason || "User left"; + if (!this.inst || this.inst.dead) { + return Promise.resolve(); // we were never connected to the network. + } + if (channel.indexOf("#") !== 0) { + return Promise.resolve(); // PM room + } + if (!this.inChannel(channel)) { + return Promise.resolve(); // we were never joined to it. + } + var self = this; + var defer = promiseutil.defer(); + this.removeChannel(channel); + self.log.debug("Leaving channel %s", channel); + this.unsafeClient.part(channel, reason, function() { + self.log.debug("Left channel %s", channel); + defer.resolve(); + }); + + return defer.promise; + } + + public inChannel(channel: string) { + return this.chanList.includes(channel); + } + + public kick(nick: string, channel: string, reason: string) { + reason = reason || "User kicked"; + if (!this.inst || this.inst.dead) { + return Promise.resolve(); // we were never connected to the network. + } + if (Object.keys(this.unsafeClient.chans).indexOf(channel) === -1) { + // we were never joined to it. We need to be joined to it to kick people. + return Promise.resolve(); + } + if (channel.indexOf("#") !== 0) { + return Promise.resolve(); // PM room + } + + return new Promise((resolve) => { + this.log.debug("Kicking %s from channel %s", nick, channel); + this.unsafeClient.send("KICK", channel, nick, reason); + resolve(); // wait for some response? Is there even one? + }); + } + + public sendAction(room, action:) { + this._keepAlive(); + let expiryTs = 0; + if (action.ts && this.server.getExpiryTimeSeconds()) { + expiryTs = action.ts + (this.server.getExpiryTimeSeconds() * 1000); + } + switch (action.type) { + case "message": + return this._sendMessage(room, "message", action.text, expiryTs); + case "notice": + return this._sendMessage(room, "notice", action.text, expiryTs); + case "emote": + return this._sendMessage(room, "action", action.text, expiryTs); + case "topic": + return this._setTopic(room, action.text); + default: + this.log.error("Unknown action type: %s", action.type); + } + return Promise.reject(new Error("Unknown action type: " + action.type)); + } + + /** + * Get the whois info for an IRC user + * @param {string} nick : The nick to call /whois on + */ + public whois(nick) { + var self = this; + return new Promise(function(resolve, reject) { + self.unsafeClient.whois(nick, function(whois) { + if (!whois.user) { + reject(new Error("Cannot find nick on whois.")); + return; + } + let idle = whois.idle ? `${whois.idle} seconds idle` : ""; + let chans = ( + (whois.channels && whois.channels.length) > 0 ? + `On channels: ${JSON.stringify(whois.channels)}` : + "" + ); + + let info = `${whois.user}@${whois.host} + Real name: ${whois.realname} + ${chans} + ${idle} + `; + resolve({ + server: self.server, + nick: nick, + msg: `Whois info for '${nick}': ${info}` + }); + }); + }); + } + + + /** + * Get the operators of a channel (including users more powerful than operators) + * @param {string} channel : The channel to call /names on + * @param {object} opts: Optional. An object containing the following key-value pairs: + * @param {string} key : Optional. The key to use to join the channel. + * @param {integer} cacheDurationMs : Optional. The duration of time to keep a + * list of operator nicks cached. If > 0, the operator nicks will be returned + * whilst the cache is still valid and it will become invalid after cacheDurationMs + * milliseconds. Cache will not be used if left undefined. + */ + public getOperators(channel, opts) { + let key = opts.key; + let cacheDurationMs = opts.cacheDurationMs; + + if (typeof key !== 'undefined' && typeof key !== 'string') { + throw new Error('key must be a string'); + } + + if (typeof cacheDurationMs !== 'undefined') { + if (!(Number.isInteger(cacheDurationMs) && cacheDurationMs > 0)) { + throw new Error('cacheDurationMs must be a positive integer'); + } + // If cached previously, use cache + if (typeof this._cachedOperatorNicksInfo[channel] !== 'undefined') { + return Promise.resolve(this._cachedOperatorNicksInfo[channel]); + } + } + + return this._joinChannel(channel, key).then(() => { + return this.getNicks(channel); + }).then((nicksInfo) => { + return this._leaveChannel(channel).then(() => nicksInfo); + }).then((nicksInfo) => { + let nicks = nicksInfo.nicks; + // RFC 1459 1.3.1: + // A channel operator is identified by the '@' symbol next to their + // nickname whenever it is associated with a channel (ie replies to the + // NAMES, WHO and WHOIS commands). + + // http://www.irc.org/tech_docs/005.html + // ISUPPORT PREFIX: + // A list of channel modes a person can get and the respective prefix a channel + // or nickname will get in case the person has it. The order of the modes goes + // from most powerful to least powerful. Those prefixes are shown in the output + // of the WHOIS, WHO and NAMES command. + // Note: Some servers only show the most powerful, others may show all of them. + + // Ergo: They are a chan op if they are "@" or "more powerful than @". + nicksInfo.operatorNicks = nicks.filter((nick) => { + for (let i = 0; i < nicksInfo.names[nick].length; i++) { + let prefix = nicksInfo.names[nick][i]; + if (prefix === "@") { + return true; + } + let cli = this.unsafeClient; + if (!cli) { + throw new Error("Missing client"); + } + if (cli.isUserPrefixMorePowerfulThan(prefix, "@")) { + return true; + } + } + return false; + }); + + if (typeof cacheDurationMs !== 'undefined') { + this._cachedOperatorNicksInfo[channel] = nicksInfo; + setTimeout(()=>{ + //Invalidate the cache + delete this._cachedOperatorNicksInfo[channel]; + }, cacheDurationMs); + } + + return nicksInfo; + }); + } + + /** + * Get the nicks of the users in a channel + * @param {string} channel : The channel to call /names on + */ + public getNicks(channel) { + var self = this; + return new Promise(function(resolve, reject) { + self.unsafeClient.names(channel, function(channelName, names) { + // names maps nicks to chan op status, where '@' indicates chan op + // names = {'nick1' : '', 'nick2' : '@', ...} + resolve({ + server: self.server, + channel: channelName, + nicks: Object.keys(names), + names: names, + }); + }); + }).timeout(5000); + } + + + /** + * Convert the given nick into a valid nick. This involves length and character + * checks on the provided nick. If the client is connected to an IRCd then the + * cmds received (e.g. NICKLEN) will be used in the calculations. If the client + * is NOT connected to an IRCd then this function will NOT take length checks + * into account. This means this function will optimistically allow long nicks + * in the hopes that it will succeed, rather than use the RFC stated maximum of + * 9 characters which is far too small. In testing, IRCds coerce long + * nicks up to the limit rather than preventing the connection entirely. + * + * This function may modify the nick in interesting ways in order to coerce the + * given nick into a valid nick. If throwOnInvalid is true, this function will + * throw a human-readable error instead of coercing the nick on invalid nicks. + * + * @param {string} nick The nick to convert into a valid nick. + * @param {boolean} throwOnInvalid True to throw an error on invalid nicks + * instead of coercing them. + * @return {string} A valid nick. + * @throws Only if throwOnInvalid is true and the nick is not a valid nick. + * The error message will contain a human-readable message which can be sent + * back to a user. + */ + private getValidNick(nick: string, throwOnInvalid: boolean): string { + // Apply a series of transformations to the nick, and check after each + // stage for mismatches to the input (and throw if appropriate). + + + // strip illegal chars according to RFC 2812 Sect 2.3.1 + let n = nick.replace(illegalCharactersRegex, ""); + if (throwOnInvalid && n !== nick) { + throw new Error(`Nick '${nick}' contains illegal characters.`); + } + + // nicks must start with a letter + if (!/^[A-Za-z]/.test(n)) { + if (throwOnInvalid) { + throw new Error(`Nick '${nick}' must start with a letter.`); + } + // Add arbitrary letter prefix. This is important for guest user + // IDs which are all numbers. + n = "M" + n; + } + + if (this.unsafeClient) { + // nicks can't be too long + let maxNickLen = 9; // RFC 1459 default + if (this.unsafeClient.supported && + typeof this.unsafeClient.supported.nicklength == "number") { + maxNickLen = this.unsafeClient.supported.nicklength; + } + if (n.length > maxNickLen) { + if (throwOnInvalid) { + throw new Error(`Nick '${nick}' is too long. (Max: ${maxNickLen})`); + } + n = n.substr(0, maxNickLen); + } + } + + return n; + } + + private keepAlive() { + this.lastActionTs = Date.now(); + var idleTimeout = this.server.getIdleTimeout(); + if (idleTimeout > 0) { + if (this.idleTimeout) { + // stop the timeout + clearTimeout(this.idleTimeout); + } + this.log.debug( + "_keepAlive; Restarting %ss idle timeout", idleTimeout + ); + // restart the timeout + var self = this; + this.idleTimeout = setTimeout(function() { + self.log.info("Idle timeout has expired"); + if (self.server.shouldSyncMembershipToIrc("initial")) { + self.log.info( + "Not disconnecting because %s is mirroring matrix membership lists", + self.server.domain + ); + return; + } + if (self.isBot) { + self.log.info("Not disconnecting because this is the bot"); + return; + } + self.disconnect( + "Idle timeout reached: " + idleTimeout + "s" + ).done(function() { + self.log.info("Idle timeout reached: Disconnected"); + }, function(e) { + self.log.error("Error when disconnecting: %s", JSON.stringify(e)); + }); + }, (1000 * idleTimeout)); + } + } + private removeChannel(channel: string) { + var i = this.chanList.indexOf(channel); + if (i === -1) { + return; + } + this.chanList.splice(i, 1); + } + + private addChannel(channel: string) { + var i = this.chanList.indexOf(channel); + if (i !== -1) { + return; // already added + } + this.chanList.push(channel); + } + + public getLastActionTs() { + return this.lastActionTs; + } + + private onConnectionCreated(connInst: ConnectionInstance, nameInfo: {username: string}) { + // listen for a connect event which is done when the TCP connection is + // established and set ident info (this is different to the connect() callback + // in node-irc which actually fires on a registered event..) + connInst.client.once("connect", function() { + var localPort = -1; + if (connInst.client.conn && connInst.client.conn.localPort) { + localPort = connInst.client.conn.localPort; + } + if (localPort > 0) { + Ident.setMapping(nameInfo.username, localPort); + } + }); + + connInst.onDisconnect = (reason) => { + this.disconnectReason = reason; + if (reason === "banned") { + // If we've been banned, this is intentional. + this.explicitDisconnect = true; + } + this.emit("client-disconnected", this); + this._eventBroker.sendMetadata(this, + "Your connection to the IRC network '" + this.server.domain + + "' has been lost. " + ); + clearTimeout(this._idleTimeout); + } + + this._eventBroker.addHooks(this, connInst); + } + + private setTopic(room, topic) { + // join the room if we haven't already + return this._joinChannel(room.channel).then(() => { + this.log.info("Setting topic to %s in channel %s", topic, room.channel); + this.unsafeClient.send("TOPIC", room.channel, topic); + }); + } + + private sendMessage(room, msgType, text, expiryTs) { + // join the room if we haven't already + var defer = promiseutil.defer(); + msgType = msgType || "message"; + this._connectDefer.promise.then(() => { + return this._joinChannel(room.channel); + }).done(() => { + // re-check timestamp to see if we should send it now + if (expiryTs && Date.now() > expiryTs) { + this.log.error(`Dropping event: too old (expired at ${expiryTs})`); + defer.resolve(); + return; + } + + if (msgType == "action") { + this.unsafeClient.action(room.channel, text); + } + else if (msgType == "notice") { + this.unsafeClient.notice(room.channel, text); + } + else if (msgType == "message") { + this.unsafeClient.say(room.channel, text); + } + defer.resolve(); + }, (e) => { + this.log.error("sendMessage: Failed to join channel " + room.channel); + defer.reject(e); + }); + return defer.promise; + } + + private joinChannel(channel, key, attemptCount) { + attemptCount = attemptCount || 1; + if (!this.unsafeClient) { + // we may be trying to join before we've connected, so check and wait + if (this._connectDefer && this._connectDefer.promise.isPending()) { + return this._connectDefer.promise.then(() => { + return this._joinChannel(channel, key, attemptCount); + }); + } + return Promise.reject(new Error("No client")); + } + if (Object.keys(this.unsafeClient.chans).indexOf(channel) !== -1) { + return Promise.resolve(new IrcRoom(this.server, channel)); + } + if (channel.indexOf("#") !== 0) { + // PM room + return Promise.resolve(new IrcRoom(this.server, channel)); + } + if (this.server.isExcludedChannel(channel)) { + return Promise.reject(new Error(channel + " is a do-not-track channel.")); + } + var defer = promiseutil.defer(); + this.log.debug("Joining channel %s", channel); + this._addChannel(channel); + var client = this.unsafeClient; + // listen for failures to join a channel (e.g. +i, +k) + var failFn = (err) => { + if (!err || !err.args) { return; } + var failCodes = [ + "err_nosuchchannel", "err_toomanychannels", "err_channelisfull", + "err_inviteonlychan", "err_bannedfromchan", "err_badchannelkey", + "err_needreggednick" + ]; + this.log.error("Join channel %s : %s", channel, JSON.stringify(err)); + if (failCodes.indexOf(err.command) !== -1 && + err.args.indexOf(channel) !== -1) { + this.log.error("Cannot track channel %s: %s", channel, err.command); + client.removeListener("error", failFn); + defer.reject(new Error(err.command)); + this.emit("join-error", this, channel, err.command); + this._eventBroker.sendMetadata( + this, `Could not join ${channel} on '${this.server.domain}': ${err.command}`, true + ); + } + } + client.once("error", failFn); + + // add a timeout to try joining again + setTimeout(() => { + if (!this.unsafeClient) { + log.error( + `Could not try to join: no client for ${this.nick}, channel = ${channel}` + ); + return; + } + // promise isn't resolved yet and we still want to join this channel + if (defer.promise.isPending() && this.chanList.indexOf(channel) !== -1) { + // we may have joined but didn't get the callback so check the client + if (Object.keys(this.unsafeClient.chans).indexOf(channel) !== -1) { + // we're joined + this.log.debug("Timed out joining %s - didn't get callback but " + + "are now joined. Resolving.", channel); + defer.resolve(new IrcRoom(this.server, channel)); + return; + } + if (attemptCount >= 5) { + defer.reject( + new Error("Failed to join " + channel + " after multiple tries") + ); + return; + } + + this.log.error("Timed out trying to join %s - trying again.", channel); + // try joining again. + attemptCount += 1; + this._joinChannel(channel, key, attemptCount).done(function(s) { + defer.resolve(s); + }, function(e) { + defer.reject(e); + }); + } + }, JOIN_TIMEOUT_MS); + + // send the JOIN with a key if it was specified. + this.unsafeClient.join(channel + (key ? " " + key : ""), () => { + this.log.debug("Joined channel %s", channel); + client.removeListener("error", failFn); + var room = new IrcRoom(this.server, channel); + defer.resolve(room); + }); + + return defer.promise; + } +} + +export const illegalCharactersRegex = /[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g; diff --git a/src/irc/ConnectionInstance.ts b/src/irc/ConnectionInstance.ts index 9219ef464..ad885b1bb 100644 --- a/src/irc/ConnectionInstance.ts +++ b/src/irc/ConnectionInstance.ts @@ -71,8 +71,10 @@ export interface ConnectionOpts { } } +export type InstanceDisconnectReason = "throttled"|"irc_error"|"net_error"|"timeout"|"raw_error"|"toomanyconns"|"banned"; + export class ConnectionInstance { - private dead: boolean = false; + public dead: boolean = false; private state: "created"|"connecting"|"connected" = "created"; private pingRateTimerId: NodeJS.Timer|null = null; private clientSidePingTimeoutTimerId: NodeJS.Timer|null = null; @@ -128,7 +130,7 @@ export class ConnectionInstance { * @param {string} reason - Reason to reject with. One of: * throttled|irc_error|net_error|timeout|raw_error|toomanyconns|banned */ - public disconnect(reason: "throttled"|"irc_error"|"net_error"|"timeout"|"raw_error"|"toomanyconns"|"banned") { + public disconnect(reason: InstanceDisconnectReason) { if (this.dead) { return Bluebird.resolve(); } diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index 99027ed42..da2a8808c 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -15,8 +15,9 @@ limitations under the License. */ import net from "net"; +import { getLogger } from "../logging"; -const log = require("../logging").get("irc-ident"); +const log = getLogger("irc-ident"); interface IdentConfig { port: number; @@ -145,5 +146,4 @@ class IdentSrv { } } -const staticInstance = new IdentSrv(); -module.exports = staticInstance; +export const staticInstance = new IdentSrv(); \ No newline at end of file diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 8430d7222..46f8e39de 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { getLogger } from "../logging"; -import * as BridgedClient from "./BridgedClient"; +import { BridgedClient, illegalCharactersRegex} from "./BridgedClient"; import { IrcClientConfig } from "../models/IrcClientConfig"; const log = getLogger("IrcServer"); @@ -462,10 +462,9 @@ export class IrcServer { } public getNick(userId: string, displayName?: string) { - const illegalChars = BridgedClient.illegalCharactersRegex; let localpart = userId.substring(1).split(":")[0]; - localpart = localpart.replace(illegalChars, ""); - displayName = displayName ? displayName.replace(illegalChars, "") : undefined; + localpart = localpart.replace(illegalCharactersRegex, ""); + displayName = displayName ? displayName.replace(illegalCharactersRegex, "") : undefined; const display = [displayName, localpart].find((n) => Boolean(n)); if (!display) { throw new Error("Could not get nick for user, all characters were invalid"); @@ -617,6 +616,7 @@ export interface IrcServerConfig { ssl?: boolean; sslselfsign?: boolean; sasl?: boolean; + password?: string; allowExpiredCerts?: boolean; additionalAddresses?: string[]; dynamicChannels: { From 16e3f696c408c71838f041103cb62e5c1417a42d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 19:48:12 +0100 Subject: [PATCH 101/350] Logging fixes --- src/irc/ClientPool.ts | 4 ++-- src/logging.ts | 24 +++++++++++++----------- src/models/BridgeRequest.ts | 18 ++++++++++-------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 03d54a1e8..e5cefecd4 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -15,14 +15,14 @@ limitations under the License. */ import * as stats from "../config/stats"; -import * as logging from "../logging"; +import { getLogger } from "../logging"; import { QueuePool } from "../util/QueuePool"; import Bluebird from "bluebird"; import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer } from "../irc/IrcServer"; import { AgeCounter, MatrixUser, MatrixRoom } from "matrix-appservice-bridge"; -const log = logging.get("ClientPool"); +const log = getLogger("ClientPool"); // We do not have these yet // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/logging.ts b/src/logging.ts index 581a6bac0..e17963cb0 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -57,18 +57,20 @@ const loggers: {[name: string]: LoggerInstance } = { let loggerTransports: TransportInstance[]; // from config +export const timestampFn = function() { + return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); +}; + +export const formatterFn = function(opts: FormatterFnOpts) { + return opts.timestamp() + ' ' + + opts.level.toUpperCase() + ':' + + (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + + (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + + (opts.meta && opts.meta.dir ? opts.meta.dir : "") + + (undefined !== opts.message ? opts.message : ''); +}; + const makeTransports = function() { - const timestampFn = function() { - return new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); - }; - const formatterFn = function(opts: FormatterFnOpts) { - return opts.timestamp() + ' ' + - opts.level.toUpperCase() + ':' + - (opts.meta && opts.meta.loggerName ? opts.meta.loggerName : "") + ' ' + - (opts.meta && opts.meta.reqId ? ("[" + opts.meta.reqId + "] ") : "") + - (opts.meta && opts.meta.dir ? opts.meta.dir : "") + - (undefined !== opts.message ? opts.message : ''); - }; let transports = []; if (loggerConfig.toConsole) { diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index f76b00885..929ba9d0e 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -14,19 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -import logging = require("../logging"); +import { getLogger, newRequestLogger } from "../logging"; import { Request } from "matrix-appservice-bridge"; -const log = logging.get("req"); - -// We do not have types for logging yet. -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type Logger = any; +import { LoggerInstance } from "winston"; +const log = getLogger("req"); export class BridgeRequest { - private log: Logger; + log: { + debug: () => void; + info: () => void; + warn: () => void; + error: () => void; + }; constructor(private req: Request) { const isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; - this.log = logging.newRequestLogger(log, req.getId(), isFromIrc); + this.log = newRequestLogger(log as LoggerInstance, req.getId(), isFromIrc); } getPromise() { From 6754db45bcf91c6b62951b18eb6343e12463232f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 23:49:04 +0100 Subject: [PATCH 102/350] Convert QuitDebouncer to Typescript --- src/bridge/IrcHandler.js | 2 +- src/bridge/QuitDebouncer.js | 142 ----------------------------------- src/bridge/QuitDebouncer.ts | 143 ++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 143 deletions(-) delete mode 100644 src/bridge/QuitDebouncer.js create mode 100644 src/bridge/QuitDebouncer.ts diff --git a/src/bridge/IrcHandler.js b/src/bridge/IrcHandler.js index c1bc4d3eb..cfa65c3c4 100644 --- a/src/bridge/IrcHandler.js +++ b/src/bridge/IrcHandler.js @@ -9,7 +9,7 @@ const MatrixUser = require("matrix-appservice-bridge").MatrixUser; const { MatrixAction } = require("../models/MatrixAction"); const { Queue } = require("../util/Queue.js"); const { QueuePool } = require("../util/QueuePool.js"); -const QuitDebouncer = require("./QuitDebouncer.js"); +const { QuitDebouncer } = require("./QuitDebouncer.js"); const RoomAccessSyncer = require("./RoomAccessSyncer.js"); const JOIN_DELAY_MS = 250; diff --git a/src/bridge/QuitDebouncer.js b/src/bridge/QuitDebouncer.js deleted file mode 100644 index bb181edcd..000000000 --- a/src/bridge/QuitDebouncer.js +++ /dev/null @@ -1,142 +0,0 @@ -/*eslint no-invalid-this: 0*/ -"use strict"; -const Promise = require("bluebird"); - -const QUIT_WAIT_DELAY_MS = 100; -const QUIT_WINDOW_MS = 1000; -const QUIT_PRESENCE = "offline"; - -function QuitDebouncer(ircBridge) { - this.ircBridge = ircBridge; - - // Measure the probability of a net-split having just happened using QUIT frequency. - // This is to smooth incoming PART spam from IRC clients that suffer from a - // net-split (or other issues that lead to mass PART-ings) - this._debouncerForServer = { - // $server.domain: { - // rejoinPromises: { - // $nick: { - // // Promise that resolves if the user joins a channel having quit - // promise: Promise, - // // Resolving function of the promise to call when a user joins - // resolve: Function - // } - // }, - // // Timestamps recorded per-server when debounceQuit is called. Old timestamps - // // are removed when a new timestamp is added. - // quitTimestampsMs:{ - // $server : [1477386173850, 1477386173825, ...] - // } - // } - }; - - // Keep a track of the times at which debounceQuit was called, and use this to - // determine the rate at which quits are being received. This can then be used - // to detect net splits. - Object.keys(this.ircBridge.config.ircService.servers).forEach((domain) => { - this._debouncerForServer[domain] = { - rejoinPromises: {}, - quitTimestampsMs: [] - }; - }); -} - -/** - * Called when the IrcHandler receives a JOIN. This resolves any promises to join that were made - * when a quit was debounced during a split. - * @param {string} nick The nick of the IRC user joining. - * @param {IrcServer} server The sending IRC server. - */ -QuitDebouncer.prototype.onJoin = function (nick, server) { - if (!this._debouncerForServer[server.domain]) { - return; - } - let rejoin = this._debouncerForServer[server.domain].rejoinPromises[nick]; - if (rejoin) { - rejoin.resolve(); - } -} - -/** - * Debounce a QUIT received by the IrcHandler to prevent net-splits from spamming leave events - * into a room when incremental membership syncing is enabled. - * @param {Request} req The metadata request. - * @param {IrcServer} server The sending IRC server. - * @param {string} matrixUser The virtual user of the user that sent QUIT. - * @param {string} nick The nick of the IRC user quiting. - * @return {Promise} which resolves to true if a leave should be sent, false otherwise. - */ -QuitDebouncer.prototype.debounceQuit = Promise.coroutine(function*(req, server, matrixUser, nick) { - // Maintain the last windowMs worth of timestamps corresponding with calls to this function. - const debouncer = this._debouncerForServer[server.domain]; - - const now = Date.now(); - debouncer.quitTimestampsMs.push(now); - - const threshold = server.getDebounceQuitsPerSecond();// Rate of quits to call net-split - - // Filter out timestamps from more than QUIT_WINDOW_MS ago - debouncer.quitTimestampsMs = debouncer.quitTimestampsMs.filter( - (t) => t > (now - QUIT_WINDOW_MS) - ); - - // Wait for a short time to allow other potential splitters to send QUITs - yield Promise.delay(QUIT_WAIT_DELAY_MS); - const isSplitOccuring = debouncer.quitTimestampsMs.length > threshold; - - // TODO: This should be replaced with "disconnected" as per matrix-appservice-irc#222 - try { - yield this.ircBridge.getAppServiceBridge().getIntent( - matrixUser.getId() - ).setPresence(QUIT_PRESENCE); - } - catch (err) { - req.log.error( - `QuitDebouncer Failed to set presence to ${QUIT_PRESENCE} for user %s: %s`, - matrixUser.getId(), - err.message - ); - } - - // Bridge QUITs if a net split is not occurring. This is in the case where a QUIT is - // received for reasons such as ping timeout or IRC client (G)UI being killed. - // We don't want to debounce users that are quiting legitimately so return early, and - // we do want to make their virtual matrix user leave the room, so return true. - if (!isSplitOccuring) { - return true; - } - - const debounceDelayMinMs = server.getQuitDebounceDelayMinMs(); - const debounceDelayMaxMs = server.getQuitDebounceDelayMaxMs(); - - const debounceMs = debounceDelayMinMs + Math.random() * ( - debounceDelayMaxMs - debounceDelayMinMs - ); - - // We do want to immediately bridge a leave if <= 0 - if (debounceMs <= 0) { - return true; - } - - req.log.info('Debouncing for ' + debounceMs + 'ms'); - - debouncer.rejoinPromises[nick] = {}; - - let p = new Promise((resolve) => { - debouncer.rejoinPromises[nick].resolve = resolve; - }).timeout(debounceMs); - debouncer.rejoinPromises[nick].promise = p; - - // Return whether the part should be bridged as a leave - try { - yield debouncer.rejoinPromises[nick].promise; - // User has joined a channel, presence has been set to online, do not leave rooms - return false; - } - catch (err) { - req.log.info("User did not rejoin (%s)", err.message); - return true; - } -}); - -module.exports = QuitDebouncer; diff --git a/src/bridge/QuitDebouncer.ts b/src/bridge/QuitDebouncer.ts new file mode 100644 index 000000000..b04f54c25 --- /dev/null +++ b/src/bridge/QuitDebouncer.ts @@ -0,0 +1,143 @@ +import Bluebird from "bluebird"; +import { IrcServer } from "../irc/IrcServer"; +import { BridgeRequest } from "../models/BridgeRequest"; +import { MatrixUser } from "matrix-appservice-bridge"; + +type IrcBridge = any; + +const QUIT_WAIT_DELAY_MS = 100; +const QUIT_WINDOW_MS = 1000; +const QUIT_PRESENCE = "offline"; + +export class QuitDebouncer { + private debouncerForServer: { + [domain: string]: { + rejoinPromises: { + [nick: string]: { + promise: Promise, + resolve: () => void, + }, + }, + quitTimestampsMs: number[] + } + } + + constructor(private ircBridge: IrcBridge) { + // Measure the probability of a net-split having just happened using QUIT frequency. + // This is to smooth incoming PART spam from IRC clients that suffer from a + // net-split (or other issues that lead to mass PART-ings) + this.debouncerForServer = {}; + + // Keep a track of the times at which debounceQuit was called, and use this to + // determine the rate at which quits are being received. This can then be used + // to detect net splits. + Object.keys(this.ircBridge.config.ircService.servers).forEach((domain) => { + this.debouncerForServer[domain] = { + rejoinPromises: {}, + quitTimestampsMs: [] + }; + }); + } + + /** + * Called when the IrcHandler receives a JOIN. This resolves any promises to join that were made + * when a quit was debounced during a split. + * @param {string} nick The nick of the IRC user joining. + * @param {IrcServer} server The sending IRC server. + */ + public onJoin(nick: string, server: IrcServer) { + if (!this.debouncerForServer[server.domain]) { + return; + } + let rejoin = this.debouncerForServer[server.domain].rejoinPromises[nick]; + if (rejoin) { + rejoin.resolve(); + } + } + + /** + * Debounce a QUIT received by the IrcHandler to prevent net-splits from spamming leave events + * into a room when incremental membership syncing is enabled. + * @param {Request} req The metadata request. + * @param {IrcServer} server The sending IRC server. + * @param {string} matrixUser The virtual user of the user that sent QUIT. + * @param {string} nick The nick of the IRC user quiting. + * @return {Promise} which resolves to true if a leave should be sent, false otherwise. + */ + public async debounceQuit (req: BridgeRequest, server: IrcServer, matrixUser: MatrixUser, nick: string) { + // Maintain the last windowMs worth of timestamps corresponding with calls to this function. + const debouncer = this.debouncerForServer[server.domain]; + + const now = Date.now(); + debouncer.quitTimestampsMs.push(now); + + const threshold = server.getDebounceQuitsPerSecond();// Rate of quits to call net-split + + // Filter out timestamps from more than QUIT_WINDOW_MS ago + debouncer.quitTimestampsMs = debouncer.quitTimestampsMs.filter( + (t) => t > (now - QUIT_WINDOW_MS) + ); + + // Wait for a short time to allow other potential splitters to send QUITs + await Bluebird.delay(QUIT_WAIT_DELAY_MS); + const isSplitOccuring = debouncer.quitTimestampsMs.length > threshold; + + // TODO: This should be replaced with "disconnected" as per matrix-appservice-irc#222 + try { + await this.ircBridge.getAppServiceBridge().getIntent( + matrixUser.getId() + ).setPresence(QUIT_PRESENCE); + } + catch (err) { + req.log.error( + `QuitDebouncer Failed to set presence to ${QUIT_PRESENCE} for user %s: %s`, + matrixUser.getId(), + err.message + ); + } + + // Bridge QUITs if a net split is not occurring. This is in the case where a QUIT is + // received for reasons such as ping timeout or IRC client (G)UI being killed. + // We don't want to debounce users that are quiting legitimately so return early, and + // we do want to make their virtual matrix user leave the room, so return true. + if (!isSplitOccuring) { + return true; + } + + const debounceDelayMinMs = server.getQuitDebounceDelayMinMs(); + const debounceDelayMaxMs = server.getQuitDebounceDelayMaxMs(); + + const debounceMs = debounceDelayMinMs + Math.random() * ( + debounceDelayMaxMs - debounceDelayMinMs + ); + + // We do want to immediately bridge a leave if <= 0 + if (debounceMs <= 0) { + return true; + } + + req.log.info('Debouncing for ' + debounceMs + 'ms'); + + let resolve: () => void; + + let promise = new Bluebird((res) => { + debouncer.rejoinPromises[nick] = { + resolve, + promise + }; + }).timeout(debounceMs); + + + + // Return whether the part should be bridged as a leave + try { + await promise; + // User has joined a channel, presence has been set to online, do not leave rooms + return false; + } + catch (err) { + req.log.info("User did not rejoin (%s)", err.message); + return true; + } + } +} \ No newline at end of file From 6d207aa7dca250050b91adb80a8a9523114d7fc5 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 23:52:29 +0100 Subject: [PATCH 103/350] Linting fixes --- src/bridge/QuitDebouncer.ts | 26 +++++++++++++------------- src/logging.ts | 3 ++- src/models/BridgeRequest.ts | 8 ++------ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/bridge/QuitDebouncer.ts b/src/bridge/QuitDebouncer.ts index b04f54c25..3c5671e9d 100644 --- a/src/bridge/QuitDebouncer.ts +++ b/src/bridge/QuitDebouncer.ts @@ -3,6 +3,8 @@ import { IrcServer } from "../irc/IrcServer"; import { BridgeRequest } from "../models/BridgeRequest"; import { MatrixUser } from "matrix-appservice-bridge"; +// We have no type for this yet. +// eslint-disable-next-line @typescript-eslint/no-explicit-any type IrcBridge = any; const QUIT_WAIT_DELAY_MS = 100; @@ -14,18 +16,18 @@ export class QuitDebouncer { [domain: string]: { rejoinPromises: { [nick: string]: { - promise: Promise, - resolve: () => void, - }, - }, - quitTimestampsMs: number[] - } - } + promise: Promise; + resolve: () => void; + }; + }; + quitTimestampsMs: number[]; + }; + }; constructor(private ircBridge: IrcBridge) { // Measure the probability of a net-split having just happened using QUIT frequency. // This is to smooth incoming PART spam from IRC clients that suffer from a - // net-split (or other issues that lead to mass PART-ings) + // net-split (or other issues that lead to mass PART-ings) this.debouncerForServer = {}; // Keep a track of the times at which debounceQuit was called, and use this to @@ -49,7 +51,7 @@ export class QuitDebouncer { if (!this.debouncerForServer[server.domain]) { return; } - let rejoin = this.debouncerForServer[server.domain].rejoinPromises[nick]; + const rejoin = this.debouncerForServer[server.domain].rejoinPromises[nick]; if (rejoin) { rejoin.resolve(); } @@ -118,9 +120,7 @@ export class QuitDebouncer { req.log.info('Debouncing for ' + debounceMs + 'ms'); - let resolve: () => void; - - let promise = new Bluebird((res) => { + const promise = new Bluebird((resolve) => { debouncer.rejoinPromises[nick] = { resolve, promise @@ -140,4 +140,4 @@ export class QuitDebouncer { return true; } } -} \ No newline at end of file +} diff --git a/src/logging.ts b/src/logging.ts index e17963cb0..d3e68b172 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -215,7 +215,8 @@ export function newRequestLogger(baseLogger: LoggerInstance, requestId: string, info: function() { decorate(baseLogger.info, arguments); }, warn: function() { decorate(baseLogger.warn, arguments); }, error: function() { decorate(baseLogger.error, arguments); }, - }; + // This is sort of untrue, but we want to have sensible types. + } as unknown as LoggerInstance; } export function setUncaughtExceptionLogger(exceptionLogger: LoggerInstance) { diff --git a/src/models/BridgeRequest.ts b/src/models/BridgeRequest.ts index 929ba9d0e..6ee5ada5b 100644 --- a/src/models/BridgeRequest.ts +++ b/src/models/BridgeRequest.ts @@ -20,14 +20,10 @@ import { LoggerInstance } from "winston"; const log = getLogger("req"); export class BridgeRequest { - log: { - debug: () => void; - info: () => void; - warn: () => void; - error: () => void; - }; + log: LoggerInstance; constructor(private req: Request) { const isFromIrc = req.getData() ? Boolean(req.getData().isFromIrc) : false; + // using "unknown" to fix odd typing. this.log = newRequestLogger(log as LoggerInstance, req.getId(), isFromIrc); } From 9f03da44ce3e25ed0aa197bef1f0de75b0e95253 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Wed, 2 Oct 2019 23:55:37 +0100 Subject: [PATCH 104/350] Newsfile --- changelog.d/830.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/830.misc diff --git a/changelog.d/830.misc b/changelog.d/830.misc new file mode 100644 index 000000000..8b2819294 --- /dev/null +++ b/changelog.d/830.misc @@ -0,0 +1 @@ +Typescriptify QuitDebouncer \ No newline at end of file From d73f09889a382e1bbcf440e3c251e1a43564b35f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:56:32 +0100 Subject: [PATCH 105/350] Update rules --- .ts.eslintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ts.eslintrc b/.ts.eslintrc index 766160200..c45b14321 100644 --- a/.ts.eslintrc +++ b/.ts.eslintrc @@ -5,6 +5,8 @@ "rules": { "@typescript-eslint/explicit-function-return-type": 0, "@typescript-eslint/camelcase": ["error", { "properties": "never" }], + "@typescript-eslint/ban-ts-ignore": 0, + "no-unused-vars": 0, // covered by @typescript-eslint/no-unused-vars "strict": ["error", "never" ], "no-var": 2 } From 6f898f19f29a2370ceb9af19fe567f3eefb93869 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:56:39 +0100 Subject: [PATCH 106/350] Convert BridgedClient to Typescript --- src/irc/BridgedClient.js | 758 --------------------------------------- src/irc/BridgedClient.ts | 452 +++++++++++++---------- 2 files changed, 253 insertions(+), 957 deletions(-) delete mode 100644 src/irc/BridgedClient.js diff --git a/src/irc/BridgedClient.js b/src/irc/BridgedClient.js deleted file mode 100644 index da5bd632c..000000000 --- a/src/irc/BridgedClient.js +++ /dev/null @@ -1,758 +0,0 @@ -/*eslint no-invalid-this: 0 */ -"use strict"; - -const Promise = require("bluebird"); -const promiseutil = require("../promiseutil"); -const util = require("util"); -const EventEmitter = require("events").EventEmitter; -const ident = require("./Ident"); -const ConnectionInstance = require("./ConnectionInstance"); -const { IrcRoom } = require("../models/IrcRoom"); -const log = require("../logging").get("BridgedClient"); - -// The length of time to wait before trying to join the channel again -const JOIN_TIMEOUT_MS = 15 * 1000; // 15s -const NICK_DELAY_TIMER_MS = 10 * 1000; // 10s - -/** - * Create a new bridged IRC client. - * @constructor - * @param {IrcServer} server - * @param {IrcClientConfig} ircClientConfig : The IRC user to create a connection for. - * @param {MatrixUser} matrixUser : Optional. The matrix user representing this virtual IRC user. - * @param {boolean} isBot : True if this is the bot - * @param {IrcEventBroker} eventBroker - * @param {IdentGenerator} identGenerator - * @param {Ipv6Generator} ipv6Generator - */ -function BridgedClient(server, ircClientConfig, matrixUser, isBot, eventBroker, identGenerator, - ipv6Generator) { - this._eventBroker = eventBroker; - this._identGenerator = identGenerator; - this._ipv6Generator = ipv6Generator; - this._clientConfig = ircClientConfig; - this.matrixUser = matrixUser; - this.server = server; - this.userId = matrixUser ? this.matrixUser.getId() : null; - this.displayName = matrixUser ? this.matrixUser.getDisplayName() : null; - this.nick = this._getValidNick( - ircClientConfig.getDesiredNick() || server.getNick(this.userId, this.displayName), - false); - this.password = ( - ircClientConfig.getPassword() ? ircClientConfig.getPassword() : server.config.password - ); - - this.isBot = Boolean(isBot); - this.lastActionTs = Date.now(); - this.inst = null; - this.instCreationFailed = false; - this.explicitDisconnect = false; - this.disconnectReason = null; - this.chanList = []; - this._connectDefer = promiseutil.defer(); - this._id = (Math.random() * 1e20).toString(36); - // decorate log lines with the nick and domain, along with an instance id - var prefix = "<" + this.nick + "@" + this.server.domain + "#" + this._id + "> "; - if (this.userId) { - prefix += "(" + this.userId + ") "; - } - this.log = { - debug: function() { - arguments[0] = prefix + arguments[0]; - log.debug.apply(log, arguments); - }, - info: function() { - arguments[0] = prefix + arguments[0]; - log.info.apply(log, arguments); - }, - error: function() { - arguments[0] = prefix + arguments[0]; - log.error.apply(log, arguments); - } - }; - - this._cachedOperatorNicksInfo = { - // $channel : info - }; -} -util.inherits(BridgedClient, EventEmitter); - -BridgedClient.prototype.getClientConfig = function() { - return this._clientConfig; -}; - -BridgedClient.prototype.kill = function(reason) { - // Nullify so that no further commands can be issued - // via unsafeClient, which should be null checked - // anyway as it is not instantiated until a connection - // has occurred. - this.unsafeClient = null; - // kill connection instance - log.info('Killing client ', this.nick); - return this.disconnect(reason || "Bridged client killed"); -} - -BridgedClient.prototype.isDead = function() { - if (this.instCreationFailed || (this.inst && this.inst.dead)) { - return true; - } - return false; -}; - -BridgedClient.prototype.toString = function() { - let domain = this.server ? this.server.domain : "NO_DOMAIN"; - return `${this.nick}@${domain}#${this._id}~${this.userId}`; -}; - -/** - * @return {ConnectionInstance} A new connected connection instance. - */ -BridgedClient.prototype.connect = Promise.coroutine(function*() { - var server = this.server; - try { - let nameInfo = yield this._identGenerator.getIrcNames( - this._clientConfig, this.matrixUser - ); - if (this.server.getIpv6Prefix()) { - // side-effects setting the IPv6 address on the client config - yield this._ipv6Generator.generate( - this.server.getIpv6Prefix(), this._clientConfig - ); - } - this.log.info( - "Connecting to IRC server %s as %s (user=%s)", - server.domain, this.nick, nameInfo.username - ); - this._eventBroker.sendMetadata(this, - `Connecting to the IRC network '${this.server.domain}' as ${this.nick}...` - ); - - let identResolver = ident.clientBegin(); - - let connInst = yield ConnectionInstance.create(server, { - nick: this.nick, - username: nameInfo.username, - realname: nameInfo.realname, - password: this.password, - // Don't use stored IPv6 addresses unless they have a prefix else they - // won't be able to turn off IPv6! - localAddress: ( - this.server.getIpv6Prefix() ? this._clientConfig.getIpv6Address() : undefined - ) - }, (inst) => { - this._onConnectionCreated(inst, nameInfo, identResolver); - }); - - this.inst = connInst; - this.unsafeClient = connInst.client; - this.emit("client-connected", this); - // we may have been assigned a different nick, so update it from source - this.nick = connInst.client.nick; - this._connectDefer.resolve(); - this._keepAlive(); - - let connectText = ( - `You've been connected to the IRC network '${this.server.domain}' as ${this.nick}.` - ); - - let userModes = this.server.getUserModes(); - if (userModes.length > 0 && !this.isBot) { - // These can fail, but the generic error listener will catch them and send them - // into the same room as the connect text, so it's probably good enough to not - // explicitly handle them. - this.unsafeClient.setUserMode("+" + userModes); - connectText += ( - ` User modes +${userModes} have been set.` - ); - } - - this._eventBroker.sendMetadata(this, connectText); - - connInst.client.addListener("nick", (old, newNick) => { - if (old === this.nick) { - this.log.info( - "NICK: Nick changed from '" + old + "' to '" + newNick + "'." - ); - this.nick = newNick; - this.emit("nick-change", this, old, newNick); - } - }); - connInst.client.addListener("error", (err) => { - // Errors we MUST notify the user about, regardless of the bridge's admin room config. - const ERRORS_TO_FORCE = ["err_nononreg"] - if (!err || !err.command || connInst.dead) { - return; - } - var msg = "Received an error on " + this.server.domain + ": " + err.command + "\n"; - msg += JSON.stringify(err.args); - this._eventBroker.sendMetadata(this, msg, ERRORS_TO_FORCE.includes(err.command)); - }); - return connInst; - } - catch (err) { - this.log.debug("Failed to connect."); - this.instCreationFailed = true; - if (identResolver) { - identResolver(); - } - throw err; - } -}); - -BridgedClient.prototype.disconnect = function(reason) { - this.explicitDisconnect = true; - if (!this.inst || this.inst.dead) { - return Promise.resolve(); - } - return this.inst.disconnect(reason); -}; - -/** - * Change this user's nick. - * @param {string} newNick The new nick for the user. - * @param {boolean} throwOnInvalid True to throw an error on invalid nicks - * instead of coercing them. - * @return {Promise} Which resolves to a message to be sent to the user. - */ -BridgedClient.prototype.changeNick = function(newNick, throwOnInvalid) { - let validNick = newNick; - try { - validNick = this._getValidNick(newNick, throwOnInvalid); - if (validNick === this.nick) { - return Promise.resolve(`Your nick is already '${validNick}'.`); - } - } - catch (err) { - return Promise.reject(err); - } - if (!this.unsafeClient) { - return Promise.reject(new Error("You are not connected to the network.")); - } - - return new Promise((resolve, reject) => { - var nickListener, nickErrListener; - var timeoutId = setTimeout(() => { - this.log.error("Timed out trying to change nick to %s", validNick); - // may have d/ced between sending nick change and now so recheck - if (this.unsafeClient) { - this.unsafeClient.removeListener("nick", nickListener); - this.unsafeClient.removeListener("error", nickErrListener); - } - reject(new Error("Timed out waiting for a response to change nick.")); - }, NICK_DELAY_TIMER_MS); - nickListener = (old, n) => { - clearTimeout(timeoutId); - this.unsafeClient.removeListener("error", nickErrListener); - resolve("Nick changed from '" + old + "' to '" + n + "'."); - }; - nickErrListener = (err) => { - if (!err || !err.command) { return; } - var failCodes = [ - "err_banonchan", "err_nickcollision", "err_nicknameinuse", - "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", - "err_nicktoofast", "err_unavailresource" - ]; - if (failCodes.indexOf(err.command) !== -1) { - this.log.error("Nick change error : %s", err.command); - clearTimeout(timeoutId); - this.unsafeClient.removeListener("nick", nickListener); - reject(new Error("Failed to change nick: " + err.command)); - } - }; - this.unsafeClient.once("nick", nickListener); - this.unsafeClient.once("error", nickErrListener); - this.unsafeClient.send("NICK", validNick); - }); -}; - -BridgedClient.prototype.joinChannel = function(channel, key) { - return this._joinChannel(channel, key); -}; - -BridgedClient.prototype.leaveChannel = function(channel, reason) { - return this._leaveChannel(channel, reason); -}; - -BridgedClient.prototype._leaveChannel = function(channel, reason) { - reason = reason || "User left"; - if (!this.inst || this.inst.dead) { - return Promise.resolve(); // we were never connected to the network. - } - if (channel.indexOf("#") !== 0) { - return Promise.resolve(); // PM room - } - if (!this.inChannel(channel)) { - return Promise.resolve(); // we were never joined to it. - } - var self = this; - var defer = promiseutil.defer(); - this._removeChannel(channel); - self.log.debug("Leaving channel %s", channel); - this.unsafeClient.part(channel, reason, function() { - self.log.debug("Left channel %s", channel); - defer.resolve(); - }); - - return defer.promise; -}; - -BridgedClient.prototype.inChannel = function(channel) { - return this.chanList.includes(channel); -} - -BridgedClient.prototype.kick = function(nick, channel, reason) { - reason = reason || "User kicked"; - if (!this.inst || this.inst.dead) { - return Promise.resolve(); // we were never connected to the network. - } - if (Object.keys(this.unsafeClient.chans).indexOf(channel) === -1) { - // we were never joined to it. We need to be joined to it to kick people. - return Promise.resolve(); - } - if (channel.indexOf("#") !== 0) { - return Promise.resolve(); // PM room - } - - return new Promise((resolve, reject) => { - this.log.debug("Kicking %s from channel %s", nick, channel); - this.unsafeClient.send("KICK", channel, nick, reason); - resolve(); // wait for some response? Is there even one? - }); -}; - -BridgedClient.prototype.sendAction = function(room, action) { - this._keepAlive(); - let expiryTs = 0; - if (action.ts && this.server.getExpiryTimeSeconds()) { - expiryTs = action.ts + (this.server.getExpiryTimeSeconds() * 1000); - } - switch (action.type) { - case "message": - return this._sendMessage(room, "message", action.text, expiryTs); - case "notice": - return this._sendMessage(room, "notice", action.text, expiryTs); - case "emote": - return this._sendMessage(room, "action", action.text, expiryTs); - case "topic": - return this._setTopic(room, action.text); - default: - this.log.error("Unknown action type: %s", action.type); - } - return Promise.reject(new Error("Unknown action type: " + action.type)); -}; - -/** - * Get the whois info for an IRC user - * @param {string} nick : The nick to call /whois on - */ -BridgedClient.prototype.whois = function(nick) { - var self = this; - return new Promise(function(resolve, reject) { - self.unsafeClient.whois(nick, function(whois) { - if (!whois.user) { - reject(new Error("Cannot find nick on whois.")); - return; - } - let idle = whois.idle ? `${whois.idle} seconds idle` : ""; - let chans = ( - (whois.channels && whois.channels.length) > 0 ? - `On channels: ${JSON.stringify(whois.channels)}` : - "" - ); - - let info = `${whois.user}@${whois.host} - Real name: ${whois.realname} - ${chans} - ${idle} - `; - resolve({ - server: self.server, - nick: nick, - msg: `Whois info for '${nick}': ${info}` - }); - }); - }); -}; - - -/** - * Get the operators of a channel (including users more powerful than operators) - * @param {string} channel : The channel to call /names on - * @param {object} opts: Optional. An object containing the following key-value pairs: - * @param {string} key : Optional. The key to use to join the channel. - * @param {integer} cacheDurationMs : Optional. The duration of time to keep a - * list of operator nicks cached. If > 0, the operator nicks will be returned - * whilst the cache is still valid and it will become invalid after cacheDurationMs - * milliseconds. Cache will not be used if left undefined. - */ -BridgedClient.prototype.getOperators = function(channel, opts) { - let key = opts.key; - let cacheDurationMs = opts.cacheDurationMs; - - if (typeof key !== 'undefined' && typeof key !== 'string') { - throw new Error('key must be a string'); - } - - if (typeof cacheDurationMs !== 'undefined') { - if (!(Number.isInteger(cacheDurationMs) && cacheDurationMs > 0)) { - throw new Error('cacheDurationMs must be a positive integer'); - } - // If cached previously, use cache - if (typeof this._cachedOperatorNicksInfo[channel] !== 'undefined') { - return Promise.resolve(this._cachedOperatorNicksInfo[channel]); - } - } - - return this._joinChannel(channel, key).then(() => { - return this.getNicks(channel); - }).then((nicksInfo) => { - return this._leaveChannel(channel).then(() => nicksInfo); - }).then((nicksInfo) => { - let nicks = nicksInfo.nicks; - // RFC 1459 1.3.1: - // A channel operator is identified by the '@' symbol next to their - // nickname whenever it is associated with a channel (ie replies to the - // NAMES, WHO and WHOIS commands). - - // http://www.irc.org/tech_docs/005.html - // ISUPPORT PREFIX: - // A list of channel modes a person can get and the respective prefix a channel - // or nickname will get in case the person has it. The order of the modes goes - // from most powerful to least powerful. Those prefixes are shown in the output - // of the WHOIS, WHO and NAMES command. - // Note: Some servers only show the most powerful, others may show all of them. - - // Ergo: They are a chan op if they are "@" or "more powerful than @". - nicksInfo.operatorNicks = nicks.filter((nick) => { - for (let i = 0; i < nicksInfo.names[nick].length; i++) { - let prefix = nicksInfo.names[nick][i]; - if (prefix === "@") { - return true; - } - let cli = this.unsafeClient; - if (!cli) { - throw new Error("Missing client"); - } - if (cli.isUserPrefixMorePowerfulThan(prefix, "@")) { - return true; - } - } - return false; - }); - - if (typeof cacheDurationMs !== 'undefined') { - this._cachedOperatorNicksInfo[channel] = nicksInfo; - setTimeout(()=>{ - //Invalidate the cache - delete this._cachedOperatorNicksInfo[channel]; - }, cacheDurationMs); - } - - return nicksInfo; - }); -}; - -/** - * Get the nicks of the users in a channel - * @param {string} channel : The channel to call /names on - */ -BridgedClient.prototype.getNicks = function(channel) { - var self = this; - return new Promise(function(resolve, reject) { - self.unsafeClient.names(channel, function(channelName, names) { - // names maps nicks to chan op status, where '@' indicates chan op - // names = {'nick1' : '', 'nick2' : '@', ...} - resolve({ - server: self.server, - channel: channelName, - nicks: Object.keys(names), - names: names, - }); - }); - }).timeout(5000); -}; - - -/** - * Convert the given nick into a valid nick. This involves length and character - * checks on the provided nick. If the client is connected to an IRCd then the - * cmds received (e.g. NICKLEN) will be used in the calculations. If the client - * is NOT connected to an IRCd then this function will NOT take length checks - * into account. This means this function will optimistically allow long nicks - * in the hopes that it will succeed, rather than use the RFC stated maximum of - * 9 characters which is far too small. In testing, IRCds coerce long - * nicks up to the limit rather than preventing the connection entirely. - * - * This function may modify the nick in interesting ways in order to coerce the - * given nick into a valid nick. If throwOnInvalid is true, this function will - * throw a human-readable error instead of coercing the nick on invalid nicks. - * - * @param {string} nick The nick to convert into a valid nick. - * @param {boolean} throwOnInvalid True to throw an error on invalid nicks - * instead of coercing them. - * @return {string} A valid nick. - * @throws Only if throwOnInvalid is true and the nick is not a valid nick. - * The error message will contain a human-readable message which can be sent - * back to a user. - */ -BridgedClient.prototype._getValidNick = function(nick, throwOnInvalid) { - // Apply a series of transformations to the nick, and check after each - // stage for mismatches to the input (and throw if appropriate). - - - // strip illegal chars according to RFC 2812 Sect 2.3.1 - let n = nick.replace(BridgedClient.illegalCharactersRegex, ""); - if (throwOnInvalid && n !== nick) { - throw new Error(`Nick '${nick}' contains illegal characters.`); - } - - // nicks must start with a letter - if (!/^[A-Za-z]/.test(n)) { - if (throwOnInvalid) { - throw new Error(`Nick '${nick}' must start with a letter.`); - } - // Add arbitrary letter prefix. This is important for guest user - // IDs which are all numbers. - n = "M" + n; - } - - if (this.unsafeClient) { - // nicks can't be too long - let maxNickLen = 9; // RFC 1459 default - if (this.unsafeClient.supported && - typeof this.unsafeClient.supported.nicklength == "number") { - maxNickLen = this.unsafeClient.supported.nicklength; - } - if (n.length > maxNickLen) { - if (throwOnInvalid) { - throw new Error(`Nick '${nick}' is too long. (Max: ${maxNickLen})`); - } - n = n.substr(0, maxNickLen); - } - } - - return n; -} - -BridgedClient.prototype._keepAlive = function() { - this.lastActionTs = Date.now(); - var idleTimeout = this.server.getIdleTimeout(); - if (idleTimeout > 0) { - if (this._idleTimeout) { - // stop the timeout - clearTimeout(this._idleTimeout); - } - this.log.debug( - "_keepAlive; Restarting %ss idle timeout", idleTimeout - ); - // restart the timeout - var self = this; - this._idleTimeout = setTimeout(function() { - self.log.info("Idle timeout has expired"); - if (self.server.shouldSyncMembershipToIrc("initial")) { - self.log.info( - "Not disconnecting because %s is mirroring matrix membership lists", - self.server.domain - ); - return; - } - if (self.isBot) { - self.log.info("Not disconnecting because this is the bot"); - return; - } - self.disconnect( - "Idle timeout reached: " + idleTimeout + "s" - ).done(function() { - self.log.info("Idle timeout reached: Disconnected"); - }, function(e) { - self.log.error("Error when disconnecting: %s", JSON.stringify(e)); - }); - }, (1000 * idleTimeout)); - } -}; -BridgedClient.prototype._removeChannel = function(channel) { - var i = this.chanList.indexOf(channel); - if (i === -1) { - return; - } - this.chanList.splice(i, 1); -}; -BridgedClient.prototype._addChannel = function(channel) { - var i = this.chanList.indexOf(channel); - if (i !== -1) { - return; // already added - } - this.chanList.push(channel); -}; -BridgedClient.prototype.getLastActionTs = function() { - return this.lastActionTs; -}; -BridgedClient.prototype._onConnectionCreated = function(connInst, nameInfo, identResolver) { - // listen for a connect event which is done when the TCP connection is - // established and set ident info (this is different to the connect() callback - // in node-irc which actually fires on a registered event..) - connInst.client.once("connect", function() { - var localPort = -1; - if (connInst.client.conn && connInst.client.conn.localPort) { - localPort = connInst.client.conn.localPort; - } - if (localPort > 0) { - ident.setMapping(nameInfo.username, localPort); - } - identResolver(); - }); - - connInst.onDisconnect = (reason) => { - this.disconnectReason = reason; - if (reason === "banned") { - // If we've been banned, this is intentional. - this.explicitDisconnect = true; - } - this.emit("client-disconnected", this); - this._eventBroker.sendMetadata(this, - "Your connection to the IRC network '" + this.server.domain + - "' has been lost. " - ); - clearTimeout(this._idleTimeout); - identResolver(); - }; - - this._eventBroker.addHooks(this, connInst); -}; - -BridgedClient.prototype._setTopic = function(room, topic) { - // join the room if we haven't already - return this._joinChannel(room.channel).then(() => { - this.log.info("Setting topic to %s in channel %s", topic, room.channel); - this.unsafeClient.send("TOPIC", room.channel, topic); - }); -} - -BridgedClient.prototype._sendMessage = function(room, msgType, text, expiryTs) { - // join the room if we haven't already - var defer = promiseutil.defer(); - msgType = msgType || "message"; - this._connectDefer.promise.then(() => { - return this._joinChannel(room.channel); - }).done(() => { - // re-check timestamp to see if we should send it now - if (expiryTs && Date.now() > expiryTs) { - this.log.error(`Dropping event: too old (expired at ${expiryTs})`); - defer.resolve(); - return; - } - - if (msgType == "action") { - this.unsafeClient.action(room.channel, text); - } - else if (msgType == "notice") { - this.unsafeClient.notice(room.channel, text); - } - else if (msgType == "message") { - this.unsafeClient.say(room.channel, text); - } - defer.resolve(); - }, (e) => { - this.log.error("sendMessage: Failed to join channel " + room.channel); - defer.reject(e); - }); - return defer.promise; -} - -BridgedClient.prototype._joinChannel = function(channel, key, attemptCount) { - attemptCount = attemptCount || 1; - if (!this.unsafeClient) { - // we may be trying to join before we've connected, so check and wait - if (this._connectDefer && this._connectDefer.promise.isPending()) { - return this._connectDefer.promise.then(() => { - return this._joinChannel(channel, key, attemptCount); - }); - } - return Promise.reject(new Error("No client")); - } - if (Object.keys(this.unsafeClient.chans).indexOf(channel) !== -1) { - return Promise.resolve(new IrcRoom(this.server, channel)); - } - if (channel.indexOf("#") !== 0) { - // PM room - return Promise.resolve(new IrcRoom(this.server, channel)); - } - if (this.server.isExcludedChannel(channel)) { - return Promise.reject(new Error(channel + " is a do-not-track channel.")); - } - var defer = promiseutil.defer(); - this.log.debug("Joining channel %s", channel); - this._addChannel(channel); - var client = this.unsafeClient; - // listen for failures to join a channel (e.g. +i, +k) - var failFn = (err) => { - if (!err || !err.args) { return; } - var failCodes = [ - "err_nosuchchannel", "err_toomanychannels", "err_channelisfull", - "err_inviteonlychan", "err_bannedfromchan", "err_badchannelkey", - "err_needreggednick" - ]; - this.log.error("Join channel %s : %s", channel, JSON.stringify(err)); - if (failCodes.indexOf(err.command) !== -1 && - err.args.indexOf(channel) !== -1) { - this.log.error("Cannot track channel %s: %s", channel, err.command); - client.removeListener("error", failFn); - defer.reject(new Error(err.command)); - this.emit("join-error", this, channel, err.command); - this._eventBroker.sendMetadata( - this, `Could not join ${channel} on '${this.server.domain}': ${err.command}`, true - ); - } - }; - client.once("error", failFn); - - // add a timeout to try joining again - setTimeout(() => { - if (!this.unsafeClient) { - log.error( - `Could not try to join: no client for ${this.nick}, channel = ${channel}` - ); - return; - } - // promise isn't resolved yet and we still want to join this channel - if (defer.promise.isPending() && this.chanList.indexOf(channel) !== -1) { - // we may have joined but didn't get the callback so check the client - if (Object.keys(this.unsafeClient.chans).indexOf(channel) !== -1) { - // we're joined - this.log.debug("Timed out joining %s - didn't get callback but " + - "are now joined. Resolving.", channel); - defer.resolve(new IrcRoom(this.server, channel)); - return; - } - if (attemptCount >= 5) { - defer.reject( - new Error("Failed to join " + channel + " after multiple tries") - ); - return; - } - - this.log.error("Timed out trying to join %s - trying again.", channel); - // try joining again. - attemptCount += 1; - this._joinChannel(channel, key, attemptCount).done(function(s) { - defer.resolve(s); - }, function(e) { - defer.reject(e); - }); - } - }, JOIN_TIMEOUT_MS); - - // send the JOIN with a key if it was specified. - this.unsafeClient.join(channel + (key ? " " + key : ""), () => { - this.log.debug("Joined channel %s", channel); - client.removeListener("error", failFn); - var room = new IrcRoom(this.server, channel); - defer.resolve(room); - }); - - return defer.promise; -} - -BridgedClient.illegalCharactersRegex = /[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g; - -module.exports = BridgedClient; diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts index dc3248d35..64e4aa82e 100644 --- a/src/irc/BridgedClient.ts +++ b/src/irc/BridgedClient.ts @@ -16,16 +16,16 @@ limitations under the License. import Bluebird from "bluebird"; import * as promiseutil from "../promiseutil"; -import util from "util"; import { EventEmitter } from "events"; -import * as Ident from "./Ident" -import { ConnectionInstance, InstanceDisconnectReason } from "./ConnectionInstance"; +import Ident from "./Ident" +import { ConnectionInstance, InstanceDisconnectReason, IrcError } from "./ConnectionInstance"; import { IrcRoom } from "../models/IrcRoom"; import { getLogger } from "../logging"; import { IrcServer } from "./IrcServer"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { MatrixUser } from "matrix-appservice-bridge"; import { LoggerInstance } from "winston"; +import { IrcAction } from "../models/IrcAction"; const log = getLogger("BridgedClient"); @@ -33,10 +33,34 @@ const log = getLogger("BridgedClient"); const JOIN_TIMEOUT_MS = 15 * 1000; // 15s const NICK_DELAY_TIMER_MS = 10 * 1000; // 10s +// All of these are not defined yet. +/* eslint-disable @typescript-eslint/no-explicit-any */ type EventBroker = any; type IdentGenerator = any; type Ipv6Generator = any; -type IrcClient = any; +type IrcClient = EventEmitter|any; +/* eslint-enable @typescript-eslint/no-explicit-any */ + +interface GetNicksResponse { + server: IrcServer; + channel: string; + nicks: string[]; + names: {[nick: string]: string}; +} + +interface GetNicksResponseOperators extends GetNicksResponse { + operatorNicks: string[]; +} + +interface WhoisResponse { + user: string; + idle: number; + channels: string[]; + host: string; + realname: string; +} + +export const illegalCharactersRegex = /[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g; export class BridgedClient extends EventEmitter { public readonly userId: string|null; @@ -44,16 +68,17 @@ export class BridgedClient extends EventEmitter { private _nick: string; private readonly id: string; private readonly password?: string; - private unsafeClient: IrcClient|null; + private _unsafeClient: IrcClient|null = null; private lastActionTs: number; - private inst: ConnectionInstance|null; - private instCreationFailed: boolean; - private explicitDisconnect: boolean; - private disconnectReason: string|null; - private chanList: string[]; + private inst: ConnectionInstance|null = null; + private instCreationFailed = false; + private _explicitDisconnect = false; + private _disconnectReason: string|null = null; + private _chanList: string[] = []; private connectDefer: promiseutil.Defer; private log: LoggerInstance; - private cachedOperatorNicksInfo: {[channel: string]: any}; + private cachedOperatorNicksInfo: {[channel: string]: GetNicksResponseOperators} = {}; + private idleTimeout: NodeJS.Timer|null = null; /** * Create a new bridged IRC client. * @constructor @@ -68,7 +93,7 @@ export class BridgedClient extends EventEmitter { constructor( public readonly server: IrcServer, private clientConfig: IrcClientConfig, - public readonly matrixUser: MatrixUser|undefined, + public readonly matrixUser: MatrixUser|undefined, public readonly isBot: boolean, private readonly eventBroker: EventBroker, private readonly identGenerator: IdentGenerator, @@ -76,66 +101,85 @@ export class BridgedClient extends EventEmitter { super(); this.userId = matrixUser ? matrixUser.getId() : null; this.displayName = matrixUser ? matrixUser.getDisplayName() : null; - this._nick = this.getValidNick( - clientConfig.getDesiredNick() || server.getNick(this.userId!, this.displayName || undefined), - false - ); + + // Set nick block + const desiredNick = clientConfig.getDesiredNick(); + let chosenNick: string|null = null; + if (desiredNick) { + chosenNick = desiredNick; + } + else if (this.userId !== null) { + chosenNick = server.getNick(this.userId, this.displayName || undefined); + } + else { + throw Error("Could not determine nick for user"); + } + this._nick = this.getValidNick(chosenNick, false); this.password = ( clientConfig.getPassword() ? clientConfig.getPassword() : server.config.password ); this.lastActionTs = Date.now(); - this.inst = null; - this.instCreationFailed = false; - this.explicitDisconnect = false; - this.disconnectReason = null; - this.chanList = []; this.connectDefer = promiseutil.defer(); this.id = (Math.random() * 1e20).toString(36); - this.unsafeClient = null; // decorate log lines with the nick and domain, along with an instance id - var prefix = "<" + this.nick + "@" + this.server.domain + "#" + this.id + "> "; + let prefix = "<" + this.nick + "@" + this.server.domain + "#" + this.id + "> "; if (this.userId) { prefix += "(" + this.userId + ") "; } this.log = { - debug: function() { - arguments[0] = prefix + arguments[0]; - log.debug.apply(log, arguments as any); + // More args magic + /* eslint-disable @typescript-eslint/no-explicit-any */ + debug: (...args: any[]) => { + const msg = prefix + args[0]; + log.debug(msg, ...args.slice(1)); }, - info: function() { - arguments[0] = prefix + arguments[0]; - log.info.apply(log, arguments as any); + info: (...args: any[]) => { + const msg = prefix + args[0]; + log.info(msg, ...args.slice(1)); }, - error: function() { - arguments[0] = prefix + arguments[0]; - log.error.apply(log, arguments as any); + error: (...args: any[]) => { + const msg = prefix + args[0]; + log.error(msg, ...args.slice(1)); } + /* eslint-enable @typescript-eslint/no-explicit-any */ } as unknown as LoggerInstance; + } - this.cachedOperatorNicksInfo = { - // $channel : info - } + public get explicitDisconnect() { + return this._explicitDisconnect; + } + + public get disconnectReason() { + return this._disconnectReason; + } + + public get chanList() { + return this._chanList; + } + + public get unsafeClient() { + return this._unsafeClient; } - public get nick() : string { + public get nick(): string { return this._nick; } - + public getClientConfig() { return this.clientConfig; } - public kill(reason: string) { + public kill(reason?: string) { // Nullify so that no further commands can be issued // via unsafeClient, which should be null checked // anyway as it is not instantiated until a connection // has occurred. - this.unsafeClient = null; + this._unsafeClient = null; // kill connection instance log.info('Killing client ', this.nick); - return this.disconnect(reason || "Bridged client killed"); + return this.disconnect("killed", reason); } public isDead() { @@ -146,7 +190,7 @@ export class BridgedClient extends EventEmitter { } public toString() { - let domain = this.server ? this.server.domain : "NO_DOMAIN"; + const domain = this.server ? this.server.domain : "NO_DOMAIN"; return `${this.nick}@${domain}#${this.id}~${this.userId}`; } @@ -155,13 +199,13 @@ export class BridgedClient extends EventEmitter { */ public async connect(): Promise { try { - let nameInfo = yield this._identGenerator.getIrcNames( - this._clientConfig, this.matrixUser + const nameInfo = await this.identGenerator.getIrcNames( + this.clientConfig, this.matrixUser ); if (this.server.getIpv6Prefix()) { // side-effects setting the IPv6 address on the client config - yield this._ipv6Generator.generate( - this.server.getIpv6Prefix(), this._clientConfig + await this.ipv6Generator.generate( + this.server.getIpv6Prefix(), this.clientConfig ); } this.log.info( @@ -172,7 +216,7 @@ export class BridgedClient extends EventEmitter { `Connecting to the IRC network '${this.server.domain}' as ${this.nick}...` ); - const connInst = yield ConnectionInstance.create(server, { + const connInst = await ConnectionInstance.create(this.server, { nick: this.nick, username: nameInfo.username, realname: nameInfo.realname, @@ -180,14 +224,14 @@ export class BridgedClient extends EventEmitter { // Don't use stored IPv6 addresses unless they have a prefix else they // won't be able to turn off IPv6! localAddress: ( - this.server.getIpv6Prefix() ? this._clientConfig.getIpv6Address() : undefined + this.server.getIpv6Prefix() ? this.clientConfig.getIpv6Address() : undefined ) - }, (inst) => { - this._onConnectionCreated(inst, nameInfo); + }, (inst: ConnectionInstance) => { + this.onConnectionCreated(inst, nameInfo); }); this.inst = connInst; - this.unsafeClient = connInst.client; + this._unsafeClient = connInst.client; this.emit("client-connected", this); // we may have been assigned a different nick, so update it from source this._nick = connInst.client.nick; @@ -198,7 +242,7 @@ export class BridgedClient extends EventEmitter { `You've been connected to the IRC network '${this.server.domain}' as ${this.nick}.` ); - let userModes = this.server.getUserModes(); + const userModes = this.server.getUserModes(); if (userModes.length > 0 && !this.isBot) { // These can fail, but the generic error listener will catch them and send them // into the same room as the connect text, so it's probably good enough to not @@ -211,7 +255,7 @@ export class BridgedClient extends EventEmitter { this.eventBroker.sendMetadata(this, connectText); - connInst.client.addListener("nick", (old, newNick) => { + connInst.client.addListener("nick", (old: string, newNick: string) => { if (old === this.nick) { this.log.info( "NICK: Nick changed from '" + old + "' to '" + newNick + "'." @@ -220,13 +264,13 @@ export class BridgedClient extends EventEmitter { this.emit("nick-change", this, old, newNick); } }); - connInst.client.addListener("error", (err) => { + connInst.client.addListener("error", (err: IrcError) => { // Errors we MUST notify the user about, regardless of the bridge's admin room config. const ERRORS_TO_FORCE = ["err_nononreg"] if (!err || !err.command || connInst.dead) { return; } - var msg = "Received an error on " + this.server.domain + ": " + err.command + "\n"; + let msg = "Received an error on " + this.server.domain + ": " + err.command + "\n"; msg += JSON.stringify(err.args); this.eventBroker.sendMetadata(this, msg, ERRORS_TO_FORCE.includes(err.command)); }); @@ -237,14 +281,25 @@ export class BridgedClient extends EventEmitter { this.instCreationFailed = true; throw err; } - }); + } + + public async reconnect() { + this.log.info( + "Reconnected %s@%s", this.nick, this.server.domain + ); + this.log.info("Rejoining %s channels", this.chanList.length); + await Promise.all(this.chanList.map((c: string) => { + return this.joinChannel(c); + })); + this.log.info("Rejoined channels"); + } - public disconnect(reason: InstanceDisconnectReason) { - this.explicitDisconnect = true; + public disconnect(reason: InstanceDisconnectReason, textReason?: string) { + this._explicitDisconnect = true; if (!this.inst || this.inst.dead) { return Promise.resolve(); } - return this.inst.disconnect(reason); + return this.inst.disconnect(reason, textReason); } /** @@ -270,7 +325,9 @@ export class BridgedClient extends EventEmitter { } return new Promise((resolve, reject) => { - let nickListener, nickErrListener; + // These are nullified to prevent the linter from thinking these should be consts. + let nickListener: ((old: string, n: string) => void) | null = null; + let nickErrListener: ((err: IrcError) => void) | null = null; const timeoutId = setTimeout(() => { this.log.error("Timed out trying to change nick to %s", validNick); // may have d/ced between sending nick change and now so recheck @@ -287,7 +344,7 @@ export class BridgedClient extends EventEmitter { } nickErrListener = (err) => { if (!err || !err.command) { return; } - var failCodes = [ + const failCodes = [ "err_banonchan", "err_nickcollision", "err_nicknameinuse", "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", "err_nicktoofast", "err_unavailresource" @@ -306,8 +363,7 @@ export class BridgedClient extends EventEmitter { } - public leaveChannel(channel: string, reason: string) { - reason = reason || "User left"; + public leaveChannel(channel: string, reason = "User left") { if (!this.inst || this.inst.dead) { return Promise.resolve(); // we were never connected to the network. } @@ -317,12 +373,11 @@ export class BridgedClient extends EventEmitter { if (!this.inChannel(channel)) { return Promise.resolve(); // we were never joined to it. } - var self = this; - var defer = promiseutil.defer(); + const defer = promiseutil.defer(); this.removeChannel(channel); - self.log.debug("Leaving channel %s", channel); - this.unsafeClient.part(channel, reason, function() { - self.log.debug("Left channel %s", channel); + this.log.debug("Leaving channel %s", channel); + this.unsafeClient.part(channel, reason, () => { + this.log.debug("Left channel %s", channel); defer.resolve(); }); @@ -353,21 +408,21 @@ export class BridgedClient extends EventEmitter { }); } - public sendAction(room, action:) { - this._keepAlive(); + public sendAction(room: IrcRoom, action: IrcAction) { + this.keepAlive(); let expiryTs = 0; if (action.ts && this.server.getExpiryTimeSeconds()) { expiryTs = action.ts + (this.server.getExpiryTimeSeconds() * 1000); } switch (action.type) { case "message": - return this._sendMessage(room, "message", action.text, expiryTs); + return this.sendMessage(room, "message", action.text, expiryTs); case "notice": - return this._sendMessage(room, "notice", action.text, expiryTs); + return this.sendMessage(room, "notice", action.text, expiryTs); case "emote": - return this._sendMessage(room, "action", action.text, expiryTs); + return this.sendMessage(room, "action", action.text, expiryTs); case "topic": - return this._setTopic(room, action.text); + return this.setTopic(room, action.text); default: this.log.error("Unknown action type: %s", action.type); } @@ -378,28 +433,27 @@ export class BridgedClient extends EventEmitter { * Get the whois info for an IRC user * @param {string} nick : The nick to call /whois on */ - public whois(nick) { - var self = this; - return new Promise(function(resolve, reject) { - self.unsafeClient.whois(nick, function(whois) { + public whois(nick: string): Promise<{ server: IrcServer; nick: string; msg: string}> { + return new Promise((resolve, reject) => { + this.unsafeClient.whois(nick, (whois: WhoisResponse) => { if (!whois.user) { reject(new Error("Cannot find nick on whois.")); return; } - let idle = whois.idle ? `${whois.idle} seconds idle` : ""; - let chans = ( + const idle = whois.idle ? `${whois.idle} seconds idle` : ""; + const chans = ( (whois.channels && whois.channels.length) > 0 ? `On channels: ${JSON.stringify(whois.channels)}` : "" ); - let info = `${whois.user}@${whois.host} + const info = `${whois.user}@${whois.host} Real name: ${whois.realname} ${chans} ${idle} `; resolve({ - server: self.server, + server: this.server, nick: nick, msg: `Whois info for '${nick}': ${info}` }); @@ -418,91 +472,94 @@ export class BridgedClient extends EventEmitter { * whilst the cache is still valid and it will become invalid after cacheDurationMs * milliseconds. Cache will not be used if left undefined. */ - public getOperators(channel, opts) { - let key = opts.key; - let cacheDurationMs = opts.cacheDurationMs; - - if (typeof key !== 'undefined' && typeof key !== 'string') { + public async getOperators(channel: string, opts: { + key?: string; + cacheDurationMs?: number; + } = {}): Promise { + const key = opts.key; + const cacheDurationMs = opts.cacheDurationMs; + + if (key !== undefined && typeof key !== 'string') { throw new Error('key must be a string'); } - if (typeof cacheDurationMs !== 'undefined') { + if (cacheDurationMs !== undefined) { if (!(Number.isInteger(cacheDurationMs) && cacheDurationMs > 0)) { throw new Error('cacheDurationMs must be a positive integer'); } // If cached previously, use cache - if (typeof this._cachedOperatorNicksInfo[channel] !== 'undefined') { - return Promise.resolve(this._cachedOperatorNicksInfo[channel]); + if (this.cachedOperatorNicksInfo[channel] !== undefined) { + return Promise.resolve(this.cachedOperatorNicksInfo[channel]); } } - - return this._joinChannel(channel, key).then(() => { - return this.getNicks(channel); - }).then((nicksInfo) => { - return this._leaveChannel(channel).then(() => nicksInfo); - }).then((nicksInfo) => { - let nicks = nicksInfo.nicks; - // RFC 1459 1.3.1: - // A channel operator is identified by the '@' symbol next to their - // nickname whenever it is associated with a channel (ie replies to the - // NAMES, WHO and WHOIS commands). - - // http://www.irc.org/tech_docs/005.html - // ISUPPORT PREFIX: - // A list of channel modes a person can get and the respective prefix a channel - // or nickname will get in case the person has it. The order of the modes goes - // from most powerful to least powerful. Those prefixes are shown in the output - // of the WHOIS, WHO and NAMES command. - // Note: Some servers only show the most powerful, others may show all of them. - - // Ergo: They are a chan op if they are "@" or "more powerful than @". - nicksInfo.operatorNicks = nicks.filter((nick) => { - for (let i = 0; i < nicksInfo.names[nick].length; i++) { - let prefix = nicksInfo.names[nick][i]; - if (prefix === "@") { - return true; - } - let cli = this.unsafeClient; - if (!cli) { - throw new Error("Missing client"); - } - if (cli.isUserPrefixMorePowerfulThan(prefix, "@")) { - return true; - } + await this.joinChannel(channel, key); + const nicksInfo = await this.getNicks(channel); + await this.leaveChannel(channel); + const nicks = nicksInfo.nicks; + // RFC 1459 1.3.1: + // A channel operator is identified by the '@' symbol next to their + // nickname whenever it is associated with a channel (ie replies to the + // NAMES, WHO and WHOIS commands). + + // http://www.irc.org/tech_docs/005.html + // ISUPPORT PREFIX: + // A list of channel modes a person can get and the respective prefix a channel + // or nickname will get in case the person has it. The order of the modes goes + // from most powerful to least powerful. Those prefixes are shown in the output + // of the WHOIS, WHO and NAMES command. + // Note: Some servers only show the most powerful, others may show all of them. + + // Ergo: They are a chan op if they are "@" or "more powerful than @". + const operatorNicks = nicks.filter((nick) => { + for (let i = 0; i < nicksInfo.names[nick].length; i++) { + const prefix = nicksInfo.names[nick][i]; + if (prefix === "@") { + return true; + } + const cli = this.unsafeClient; + if (!cli) { + throw new Error("Missing client"); + } + if (cli.isUserPrefixMorePowerfulThan(prefix, "@")) { + return true; } - return false; - }); - - if (typeof cacheDurationMs !== 'undefined') { - this._cachedOperatorNicksInfo[channel] = nicksInfo; - setTimeout(()=>{ - //Invalidate the cache - delete this._cachedOperatorNicksInfo[channel]; - }, cacheDurationMs); } - - return nicksInfo; + return false; }); + + const nicksInfoExtended = { + ...nicksInfo, + operatorNicks + }; + + if (typeof cacheDurationMs !== 'undefined') { + this.cachedOperatorNicksInfo[channel] = nicksInfoExtended; + setTimeout(()=>{ + //Invalidate the cache + delete this.cachedOperatorNicksInfo[channel]; + }, cacheDurationMs); + } + + return nicksInfoExtended; } /** * Get the nicks of the users in a channel * @param {string} channel : The channel to call /names on */ - public getNicks(channel) { - var self = this; - return new Promise(function(resolve, reject) { - self.unsafeClient.names(channel, function(channelName, names) { + public getNicks(channel: string): Bluebird { + return new Bluebird((resolve) => { + this.unsafeClient.names(channel, (channelName: string, names: {[nick: string]: string}) => { // names maps nicks to chan op status, where '@' indicates chan op // names = {'nick1' : '', 'nick2' : '@', ...} resolve({ - server: self.server, + server: this.server, channel: channelName, nicks: Object.keys(names), names: names, }); }); - }).timeout(5000); + }).timeout(5000) as Bluebird; } @@ -569,7 +626,7 @@ export class BridgedClient extends EventEmitter { private keepAlive() { this.lastActionTs = Date.now(); - var idleTimeout = this.server.getIdleTimeout(); + const idleTimeout = this.server.getIdleTimeout(); if (idleTimeout > 0) { if (this.idleTimeout) { // stop the timeout @@ -579,32 +636,31 @@ export class BridgedClient extends EventEmitter { "_keepAlive; Restarting %ss idle timeout", idleTimeout ); // restart the timeout - var self = this; - this.idleTimeout = setTimeout(function() { - self.log.info("Idle timeout has expired"); - if (self.server.shouldSyncMembershipToIrc("initial")) { - self.log.info( + this.idleTimeout = setTimeout(() => { + this.log.info("Idle timeout has expired"); + if (this.server.shouldSyncMembershipToIrc("initial")) { + this.log.info( "Not disconnecting because %s is mirroring matrix membership lists", - self.server.domain + this.server.domain ); return; } - if (self.isBot) { - self.log.info("Not disconnecting because this is the bot"); + if (this.isBot) { + this.log.info("Not disconnecting because this is the bot"); return; } - self.disconnect( - "Idle timeout reached: " + idleTimeout + "s" - ).done(function() { - self.log.info("Idle timeout reached: Disconnected"); - }, function(e) { - self.log.error("Error when disconnecting: %s", JSON.stringify(e)); + this.disconnect( + "idle", `Idle timeout reached: ${idleTimeout}s` + ).then(() => { + this.log.info("Idle timeout reached: Disconnected"); + }).catch((e) => { + this.log.error("Error when disconnecting: %s", JSON.stringify(e)); }); }, (1000 * idleTimeout)); } } private removeChannel(channel: string) { - var i = this.chanList.indexOf(channel); + const i = this.chanList.indexOf(channel); if (i === -1) { return; } @@ -612,7 +668,7 @@ export class BridgedClient extends EventEmitter { } private addChannel(channel: string) { - var i = this.chanList.indexOf(channel); + const i = this.chanList.indexOf(channel); if (i !== -1) { return; // already added } @@ -628,7 +684,7 @@ export class BridgedClient extends EventEmitter { // established and set ident info (this is different to the connect() callback // in node-irc which actually fires on a registered event..) connInst.client.once("connect", function() { - var localPort = -1; + let localPort = -1; if (connInst.client.conn && connInst.client.conn.localPort) { localPort = connInst.client.conn.localPort; } @@ -638,37 +694,38 @@ export class BridgedClient extends EventEmitter { }); connInst.onDisconnect = (reason) => { - this.disconnectReason = reason; + this._disconnectReason = reason; if (reason === "banned") { // If we've been banned, this is intentional. - this.explicitDisconnect = true; + this._explicitDisconnect = true; } this.emit("client-disconnected", this); - this._eventBroker.sendMetadata(this, + this.eventBroker.sendMetadata(this, "Your connection to the IRC network '" + this.server.domain + "' has been lost. " ); - clearTimeout(this._idleTimeout); + if (this.idleTimeout) { + clearTimeout(this.idleTimeout); + } } - this._eventBroker.addHooks(this, connInst); + this.eventBroker.addHooks(this, connInst); } - private setTopic(room, topic) { + private async setTopic(room: IrcRoom, topic: string): Promise { // join the room if we haven't already - return this._joinChannel(room.channel).then(() => { - this.log.info("Setting topic to %s in channel %s", topic, room.channel); - this.unsafeClient.send("TOPIC", room.channel, topic); - }); + await this.joinChannel(room.channel) + this.log.info("Setting topic to %s in channel %s", topic, room.channel); + this.unsafeClient.send("TOPIC", room.channel, topic); } - private sendMessage(room, msgType, text, expiryTs) { + private async sendMessage(room: IrcRoom, msgType: string, text: string, expiryTs: number) { // join the room if we haven't already - var defer = promiseutil.defer(); + const defer = promiseutil.defer(); msgType = msgType || "message"; - this._connectDefer.promise.then(() => { - return this._joinChannel(room.channel); - }).done(() => { + try { + await this.connectDefer.promise; + await this.joinChannel(room.channel); // re-check timestamp to see if we should send it now if (expiryTs && Date.now() > expiryTs) { this.log.error(`Dropping event: too old (expired at ${expiryTs})`); @@ -686,54 +743,53 @@ export class BridgedClient extends EventEmitter { this.unsafeClient.say(room.channel, text); } defer.resolve(); - }, (e) => { + } + catch (ex) { this.log.error("sendMessage: Failed to join channel " + room.channel); - defer.reject(e); - }); - return defer.promise; + defer.reject(ex); + } + await defer.promise; } - private joinChannel(channel, key, attemptCount) { - attemptCount = attemptCount || 1; + private joinChannel(channel: string, key?: string, attemptCount = 1): Bluebird { if (!this.unsafeClient) { // we may be trying to join before we've connected, so check and wait - if (this._connectDefer && this._connectDefer.promise.isPending()) { - return this._connectDefer.promise.then(() => { - return this._joinChannel(channel, key, attemptCount); + if (this.connectDefer && this.connectDefer.promise.isPending()) { + return this.connectDefer.promise.then(() => { + return this.joinChannel(channel, key, attemptCount); }); } - return Promise.reject(new Error("No client")); + return Bluebird.reject(new Error("No client")); } if (Object.keys(this.unsafeClient.chans).indexOf(channel) !== -1) { - return Promise.resolve(new IrcRoom(this.server, channel)); + return Bluebird.resolve(new IrcRoom(this.server, channel)); } if (channel.indexOf("#") !== 0) { // PM room - return Promise.resolve(new IrcRoom(this.server, channel)); + return Bluebird.resolve(new IrcRoom(this.server, channel)); } if (this.server.isExcludedChannel(channel)) { - return Promise.reject(new Error(channel + " is a do-not-track channel.")); + return Bluebird.reject(new Error(channel + " is a do-not-track channel.")); } - var defer = promiseutil.defer(); + const defer = promiseutil.defer() as promiseutil.Defer; this.log.debug("Joining channel %s", channel); - this._addChannel(channel); - var client = this.unsafeClient; + this.addChannel(channel); + const client = this.unsafeClient; // listen for failures to join a channel (e.g. +i, +k) - var failFn = (err) => { + const failFn = (err: IrcError) => { if (!err || !err.args) { return; } - var failCodes = [ + const failCodes = [ "err_nosuchchannel", "err_toomanychannels", "err_channelisfull", "err_inviteonlychan", "err_bannedfromchan", "err_badchannelkey", "err_needreggednick" ]; this.log.error("Join channel %s : %s", channel, JSON.stringify(err)); - if (failCodes.indexOf(err.command) !== -1 && - err.args.indexOf(channel) !== -1) { + if (err.command && failCodes.includes(err.command) && err.args.includes(channel)) { this.log.error("Cannot track channel %s: %s", channel, err.command); client.removeListener("error", failFn); defer.reject(new Error(err.command)); this.emit("join-error", this, channel, err.command); - this._eventBroker.sendMetadata( + this.eventBroker.sendMetadata( this, `Could not join ${channel} on '${this.server.domain}': ${err.command}`, true ); } @@ -768,9 +824,9 @@ export class BridgedClient extends EventEmitter { this.log.error("Timed out trying to join %s - trying again.", channel); // try joining again. attemptCount += 1; - this._joinChannel(channel, key, attemptCount).done(function(s) { + this.joinChannel(channel, key, attemptCount).then((s) => { defer.resolve(s); - }, function(e) { + }).catch((e: Error) => { defer.reject(e); }); } @@ -780,12 +836,10 @@ export class BridgedClient extends EventEmitter { this.unsafeClient.join(channel + (key ? " " + key : ""), () => { this.log.debug("Joined channel %s", channel); client.removeListener("error", failFn); - var room = new IrcRoom(this.server, channel); + const room = new IrcRoom(this.server, channel); defer.resolve(room); }); return defer.promise; } } - -export const illegalCharactersRegex = /[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g; From 48c776edc73f7a9def9ebe946367534847d1f6d6 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:57:21 +0100 Subject: [PATCH 107/350] Update imports where needed --- src/DebugApi.ts | 3 ++- src/bridge/IrcBridge.js | 2 +- src/irc/ClientPool.ts | 3 +-- src/irc/IrcServer.ts | 2 +- src/irc/Scheduler.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DebugApi.ts b/src/DebugApi.ts index 44f400cc1..01ee5d489 100644 --- a/src/DebugApi.ts +++ b/src/DebugApi.ts @@ -17,13 +17,14 @@ limitations under the License. import querystring, { ParsedUrlQuery } from "querystring"; import Bluebird from "bluebird"; import http, { IncomingMessage, ServerResponse } from "http"; -import { IrcServer } from "./irc/IrcServer"; +import { IrcServer } from "./irc/IrcServer"; import { BridgeRequest } from "./models/BridgeRequest"; import { inspect } from "util"; import { DataStore } from "./datastore/DataStore"; import { ClientPool } from "./irc/ClientPool"; import { getLogger } from "./logging"; +import { BridgedClient } from "./irc/BridgedClient"; const log = getLogger("DebugApi"); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 029bc6842..6cad62ecf 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -12,7 +12,7 @@ var Ipv6Generator = require("../irc/Ipv6Generator.js"); const { IrcServer } = require("../irc/IrcServer.js"); const { ClientPool } = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); -var BridgedClient = require("../irc/BridgedClient"); +const { BridgedClient} = require("../irc/BridgedClient"); const { IrcUser } = require("../models/IrcUser"); const { IrcRoom } = require("../models/IrcRoom"); const { IrcClientConfig } = require("../models/IrcClientConfig"); diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index e5cefecd4..229302d9c 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -22,12 +22,11 @@ import { BridgeRequest } from "../models/BridgeRequest"; import { IrcClientConfig } from "../models/IrcClientConfig"; import { IrcServer } from "../irc/IrcServer"; import { AgeCounter, MatrixUser, MatrixRoom } from "matrix-appservice-bridge"; +import { BridgedClient } from "./BridgedClient"; const log = getLogger("ClientPool"); // We do not have these yet // eslint-disable-next-line @typescript-eslint/no-explicit-any -type BridgedClient = any; -// eslint-disable-next-line @typescript-eslint/no-explicit-any type IrcBridge = any; interface ReconnectionItem { diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 46f8e39de..51dc6b44b 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { getLogger } from "../logging"; -import { BridgedClient, illegalCharactersRegex} from "./BridgedClient"; +import { illegalCharactersRegex} from "./BridgedClient"; import { IrcClientConfig } from "../models/IrcClientConfig"; const log = getLogger("IrcServer"); diff --git a/src/irc/Scheduler.ts b/src/irc/Scheduler.ts index 2ae67e5e2..951878e70 100644 --- a/src/irc/Scheduler.ts +++ b/src/irc/Scheduler.ts @@ -15,7 +15,7 @@ limitations under the License. */ import Bluebird from "bluebird"; -import{ getLogger } from "../logging"; +import { getLogger } from "../logging"; import { Queue } from "../util/Queue"; import { IrcServer } from "./IrcServer"; From 508804c26918a7a703f8b026b4c9f5b4a2fa26a1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:58:53 +0100 Subject: [PATCH 108/350] Various linting tweaks --- src/DebugApi.ts | 41 +++++++++++---------- src/datastore/postgres/PgDataStore.ts | 51 ++++++++++++++------------- src/irc/Ident.ts | 2 +- src/irc/IrcServer.ts | 4 +-- src/promiseutil.ts | 12 +++---- src/util/Queue.ts | 2 +- 6 files changed, 57 insertions(+), 55 deletions(-) diff --git a/src/DebugApi.ts b/src/DebugApi.ts index 01ee5d489..3bd24e24e 100644 --- a/src/DebugApi.ts +++ b/src/DebugApi.ts @@ -30,8 +30,6 @@ const log = getLogger("DebugApi"); // eslint-disable-next-line @typescript-eslint/no-explicit-any type IrcBridge = any; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type BridgedClient = any; export class DebugApi { constructor(private ircBridge: IrcBridge, private port: number, private servers: IrcServer[], private pool: ClientPool, private token: string) { @@ -40,11 +38,12 @@ export class DebugApi { public run () { log.info("DEBUG API LISTENING ON :%d", this.port); - + http.createServer((req, res) => { try { this.onRequest(req, res); - } catch (err) { + } + catch (err) { if (!res.finished) { res.end(); } @@ -86,7 +85,7 @@ export class DebugApi { } // Looks like /irc/$domain/user/$user_id - let segs = path.split("/"); + const segs = path.split("/"); if (segs.length !== 5 || segs[1] !== "irc" || segs[3] !== "user") { response.writeHead(404, {"Content-Type": "text/plain"}); response.write("Not a valid debug path.\n"); @@ -94,13 +93,13 @@ export class DebugApi { return; } - let domain = segs[2]; - let user = segs[4]; + const domain = segs[2]; + const user = segs[4]; log.debug("Domain: %s User: %s", domain, user); const server = this.servers.find((s) => s.domain === domain); - + if (server === undefined) { response.writeHead(400, {"Content-Type": "text/plain"}); response.write("Not a valid domain.\n"); @@ -205,7 +204,7 @@ export class DebugApi { if (!user) { return this.pool.getBot(server); } - return this.pool.getBridgedClientByUserId(server, user); + return this.pool.getBridgedClientByUserId(server, user); } private getClientState(server: IrcServer, user: string) { @@ -266,7 +265,7 @@ export class DebugApi { private async killPortal (req: IncomingMessage, response: ServerResponse) { const store = this.ircBridge.getStore() as DataStore; - const result: { error: string[], stages: string[] } = { + const result: { error: string[]; stages: string[] } = { error: [], // string|[string] containing a fatal error or minor errors. stages: [] // stages completed for removing the room. It's possible it might only // half complete, and we should make that obvious. @@ -306,14 +305,14 @@ export class DebugApi { this.wrapJsonResponse(result.error, false, response); return; } - + log.warn( `Requested deletion of portal room alias ${roomId} through debug API Domain: ${domain} Channel: ${channel} Leave Notice: ${notice} Remove Alias: ${remove_alias}`); - + // Find room const room = await store.getRoom( roomId, @@ -326,14 +325,14 @@ export class DebugApi { this.wrapJsonResponse(result, false, response); return; } - + const server = this.servers.find((srv) => srv.domain === domain); if (server === undefined) { result.error.push("Server not found!"); this.wrapJsonResponse(result, false, response); return; } - + // Drop room from room store. await store.removeRoom( roomId, @@ -342,7 +341,7 @@ export class DebugApi { "alias" ); result.stages.push("Removed room from store"); - + if (notice) { try { await this.ircBridge.getAppServiceBridge().getIntent().sendEvent(roomId, "notice", @@ -355,7 +354,7 @@ export class DebugApi { result.error.push("Failed to send a leave notice"); } } - + if (remove_alias) { const roomAlias = server.getAliasFromChannel(channel); try { @@ -366,7 +365,7 @@ export class DebugApi { result.error.push("Failed to remove alias"); } } - + // Drop clients from room. // The provisioner will only drop clients who are not in other rooms. // It will also leave the MatrixBot. @@ -384,7 +383,7 @@ export class DebugApi { this.wrapJsonResponse(result, false, response); return; } - + result.stages.push("Parted clients where applicable."); this.wrapJsonResponse(result, true, response); } @@ -422,8 +421,8 @@ export class DebugApi { "info": String(ex), }, false, response); } - }; - + } + private wrapJsonReq (req: IncomingMessage, response: ServerResponse): Bluebird { let body = ""; req.on("data", (chunk) => { body += chunk; }); @@ -444,7 +443,7 @@ export class DebugApi { }); }); } - + private wrapJsonResponse (json: unknown, isOk: boolean, response: ServerResponse) { response.writeHead(isOk === true ? 200 : 500, {"Content-Type": "application/json"}); response.write(JSON.stringify(json)); diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 73a6c172c..08ecc3af2 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -36,10 +36,10 @@ export class PgDataStore implements DataStore { public static readonly LATEST_SCHEMA = 1; private pgPool: Pool; - private hasEnded: boolean = false; + private hasEnded = false; private cryptoStore?: StringCrypto; - constructor(private bridgeDomain: string, connectionString: string, pkeyPath?: string, min: number = 1, max: number = 4) { + constructor(private bridgeDomain: string, connectionString: string, pkeyPath?: string, min = 1, max = 4) { this.pgPool = new Pool({ connectionString, min, @@ -58,7 +58,7 @@ export class PgDataStore implements DataStore { } // Ensure we clean up on exit this.pgPool.end(); - }) + }) } public async setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise { @@ -112,7 +112,7 @@ export class PgDataStore implements DataStore { irc_json: ircJson, matrix_json: matrixJson, }; - const statement = PgDataStore.BuildUpsertStatement("rooms","ON CONSTRAINT cons_rooms_unique", Object.keys(parameters)); + const statement = PgDataStore.BuildUpsertStatement("rooms", "ON CONSTRAINT cons_rooms_unique", Object.keys(parameters)); await this.pgPool.query(statement, Object.values(parameters)); } @@ -120,7 +120,7 @@ export class PgDataStore implements DataStore { return { id: "", matrix: new MatrixRoom(pgEntry.room_id, pgEntry.matrix_json), - remote: new RemoteRoom("", + remote: new RemoteRoom("", { ...pgEntry.irc_json, channel: pgEntry.irc_channel, @@ -170,7 +170,7 @@ export class PgDataStore implements DataStore { channel: e.irc_channel, }); }) - + return mappings; } @@ -194,10 +194,10 @@ export class PgDataStore implements DataStore { } public async getIrcChannelsForRoomId(roomId: string): Promise { - let entries = await this.pgPool.query("SELECT irc_domain, irc_channel FROM rooms WHERE room_id = $1", [ roomId ]); + let entries = await this.pgPool.query("SELECT irc_domain, irc_channel FROM rooms WHERE room_id = $1", [roomId]); if (entries.rowCount === 0) { // Could be a PM room, if it's not a channel. - entries = await this.pgPool.query("SELECT irc_domain, irc_nick FROM pm_rooms WHERE room_id = $1", [ roomId ]); + entries = await this.pgPool.query("SELECT irc_domain, irc_nick FROM pm_rooms WHERE room_id = $1", [roomId]); } return entries.rows.map((e) => { const server = this.serverMappings[e.irc_domain]; @@ -209,11 +209,11 @@ export class PgDataStore implements DataStore { }).filter((i) => i !== undefined); } - public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{ [roomId: string]: IrcRoom[]; }> { + public async getIrcChannelsForRoomIds(roomIds: string[]): Promise<{ [roomId: string]: IrcRoom[] }> { const entries = await this.pgPool.query("SELECT room_id, irc_domain, irc_channel FROM rooms WHERE room_id IN $1", [ roomIds ]); - const mapping: { [roomId: string]: IrcRoom[]; } = {}; + const mapping: { [roomId: string]: IrcRoom[] } = {}; entries.rows.forEach((e) => { const server = this.serverMappings[e.irc_domain]; if (!server) { @@ -253,7 +253,7 @@ export class PgDataStore implements DataStore { return entries.rows.map((e) => PgDataStore.pgToRoomEntry(e)); } - public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string[]; }> { + public async getModesForChannel(server: IrcServer, channel: string): Promise<{ [id: string]: string[] }> { log.debug(`Getting modes for ${server.domain} ${channel}`); const mapping: {[id: string]: string[]} = {}; const entries = await this.pgPool.query( @@ -334,7 +334,7 @@ export class PgDataStore implements DataStore { return []; } log.info(`Fetching all channels for ${domain}`); - const chanSet = await this.pgPool.query("SELECT DISTINCT irc_channel FROM rooms WHERE irc_domain = $1", [ domain ]); + const chanSet = await this.pgPool.query("SELECT DISTINCT irc_channel FROM rooms WHERE irc_domain = $1", [domain]); return chanSet.rows.map((e) => e.irc_channel as string); } @@ -354,7 +354,7 @@ export class PgDataStore implements DataStore { } public async setIpv6Counter(counter: number): Promise { - await this.pgPool.query("UPDATE ipv6_counter SET count = $1", [ counter ]); + await this.pgPool.query("UPDATE ipv6_counter SET count = $1", [counter]); } public async upsertMatrixRoom(room: MatrixRoom): Promise { @@ -368,7 +368,7 @@ export class PgDataStore implements DataStore { } public async getAdminRoomById(roomId: string): Promise { - const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE room_id = $1", [ roomId ]); + const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE room_id = $1", [roomId]); if (res.rowCount === 0) { return null; } @@ -379,11 +379,11 @@ export class PgDataStore implements DataStore { await this.pgPool.query(PgDataStore.BuildUpsertStatement("admin_rooms", "(room_id)", [ "room_id", "user_id", - ]), [ room.getId(), userId ]); + ]), [room.getId(), userId]); } public async getAdminRoomByUserId(userId: string): Promise { - const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE user_id = $1", [ userId ]); + const res = await this.pgPool.query("SELECT room_id FROM admin_rooms WHERE user_id = $1", [userId]); if (res.rowCount === 0) { return null; } @@ -400,7 +400,7 @@ export class PgDataStore implements DataStore { } public async getIrcClientConfig(userId: string, domain: string): Promise { - const res = await this.pgPool.query("SELECT config, password FROM client_config WHERE user_id = $1 and domain = $2", + const res = await this.pgPool.query("SELECT config, password FROM client_config WHERE user_id = $1 and domain = $2", [ userId, domain @@ -453,7 +453,7 @@ export class PgDataStore implements DataStore { public async getUserFeatures(userId: string): Promise { const pgRes = ( await this.pgPool.query("SELECT features FROM user_features WHERE user_id = $1", - [ userId ]) + [userId]) ); if (pgRes.rowCount === 0) { return {}; @@ -469,7 +469,7 @@ export class PgDataStore implements DataStore { await this.pgPool.query(statement, [userId, JSON.stringify(features)]); } - public async storePass(userId: string, domain: string, pass: string, encrypt: boolean = true): Promise { + public async storePass(userId: string, domain: string, pass: string, encrypt = true): Promise { let password = pass; if (encrypt) { if (!this.cryptoStore) { @@ -498,7 +498,8 @@ export class PgDataStore implements DataStore { ); if (res.rowCount === 0) { return; - } else if (res.rowCount > 1) { + } + else if (res.rowCount > 1) { log.error("getMatrixUserByUsername returned %s results for %s on %s", res.rowCount, username, domain); } return new MatrixUser(res.rows[0].user_id, res.rows[0].data); @@ -514,7 +515,8 @@ export class PgDataStore implements DataStore { await runSchema(this.pgPool); currentVersion++; await this.updateSchemaVersion(currentVersion); - } catch (ex) { + } + catch (ex) { log.warn(`Failed to run schema v${currentVersion + 1}:`, ex); throw Error("Failed to update database schema"); } @@ -536,14 +538,15 @@ export class PgDataStore implements DataStore { private async updateSchemaVersion(version: number) { log.debug(`updateSchemaVersion: ${version}`); - await this.pgPool.query("UPDATE schema SET version = $1;", [ version ]); + await this.pgPool.query("UPDATE schema SET version = $1;", [version]); } private async getSchemaVersion(): Promise { try { const { rows } = await this.pgPool.query("SELECT version FROM SCHEMA"); return rows[0].version; - } catch (ex) { + } + catch (ex) { if (ex.code === "42P01") { // undefined_table log.warn("Schema table could not be found"); return 0; @@ -560,4 +563,4 @@ export class PgDataStore implements DataStore { const statement = `INSERT INTO ${table} (${keys}) VALUES (${keysValues}) ON CONFLICT ${constraint} DO UPDATE SET ${keysSets}`; return statement; } -} \ No newline at end of file +} diff --git a/src/irc/Ident.ts b/src/irc/Ident.ts index da2a8808c..7ad95bef5 100644 --- a/src/irc/Ident.ts +++ b/src/irc/Ident.ts @@ -146,4 +146,4 @@ class IdentSrv { } } -export const staticInstance = new IdentSrv(); \ No newline at end of file +export default new IdentSrv(); diff --git a/src/irc/IrcServer.ts b/src/irc/IrcServer.ts index 51dc6b44b..83bf68aac 100644 --- a/src/irc/IrcServer.ts +++ b/src/irc/IrcServer.ts @@ -315,7 +315,7 @@ export class IrcServer { return this.config.privateMessages.enabled; } - public shouldSyncMembershipToIrc(kind: MembershipSyncKind, roomId: string) { + public shouldSyncMembershipToIrc(kind: MembershipSyncKind, roomId?: string) { return this._shouldSyncMembership(kind, roomId, true); } @@ -323,7 +323,7 @@ export class IrcServer { return this._shouldSyncMembership(kind, channel, false); } - public _shouldSyncMembership(kind: MembershipSyncKind, identifier: string, toIrc: boolean) { + public _shouldSyncMembership(kind: MembershipSyncKind, identifier: string|undefined, toIrc: boolean) { if (["incremental", "initial"].indexOf(kind) === -1) { throw new Error("Bad kind: " + kind); } diff --git a/src/promiseutil.ts b/src/promiseutil.ts index be2fc0c96..f76b0a79d 100644 --- a/src/promiseutil.ts +++ b/src/promiseutil.ts @@ -19,20 +19,20 @@ import Bluebird from "bluebird"; export interface Defer { resolve: (value?: T) => void; reject: (err?: unknown) => void; - promise: Promise; + promise: Bluebird; } export function defer(): Defer { - let resolve: (value?: T) => void; - let reject: (err?: unknown) => void; + let resolve!: (value?: T) => void; + let reject!: (err?: unknown) => void; const promise = new Bluebird((res, rej) => { resolve = res; reject = rej }); return { - resolve: resolve!, - reject: reject!, - promise: promise + resolve: resolve, + reject: reject, + promise: promise as Bluebird }; } diff --git a/src/util/Queue.ts b/src/util/Queue.ts index 9bcf72862..8beccb59e 100644 --- a/src/util/Queue.ts +++ b/src/util/Queue.ts @@ -24,7 +24,7 @@ export interface QueueItem { defer: Defer; } -export type QueueProcessFn = (item: unknown) => Bluebird|void; +export type QueueProcessFn = (item: unknown) => Promise|void; export class Queue { private queue: QueueItem[] = []; From 05c4214cf23825dd8d858f6ffc5e587c177f8606 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:59:15 +0100 Subject: [PATCH 109/350] Tweak ClientPool for BridgedClient types --- src/irc/ClientPool.ts | 94 ++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 229302d9c..57e955d16 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -39,10 +39,10 @@ interface ReconnectionItem { * and may be closed for a variety of reasons. */ export class ClientPool { - private botClients: { [serverDomain: string]: BridgedClient}; + private botClients: { [serverDomain: string]: BridgedClient|undefined}; private virtualClients: { [serverDomain: string]: { - nicks: { [nickname: string]: BridgedClient}; - userIds: { [userId: string]: BridgedClient}; + nicks: { [nickname: string]: BridgedClient|undefined}; + userIds: { [userId: string]: BridgedClient|undefined}; pending: { [nick: string]: BridgedClient}; };}; private virtualClientCounts: { [serverDomain: string]: number }; @@ -79,7 +79,7 @@ export class ClientPool { public killAllClients(): Bluebird { const domainList = Object.keys(this.virtualClients); - let clients: BridgedClient[] = []; + let clients: (BridgedClient|undefined)[] = []; domainList.forEach((domain) => { clients = clients.concat( Object.keys(this.virtualClients[domain].nicks).map( @@ -96,10 +96,10 @@ export class ClientPool { clients.push(this.botClients[domain]); }); - clients = clients.filter((c) => Boolean(c)); + const safeClients = clients.filter((c) => Boolean(c)) as BridgedClient[]; return Bluebird.all( - clients.map( + safeClients.map( (client) => client.kill() ) ); @@ -218,7 +218,7 @@ export class ClientPool { public getBridgedClientsForRegex(userIdRegexString: string) { const userIdRegex = new RegExp(userIdRegexString); const domainList = Object.keys(this.virtualClients); - const clientList: {[userId: string]: BridgedClient} = {}; + const clientList: {[userId: string]: BridgedClient[]} = {}; domainList.forEach((domain) => { Object.keys( this.virtualClients[domain].userIds @@ -228,14 +228,17 @@ export class ClientPool { if (!clientList[userId]) { clientList[userId] = []; } - clientList[userId].push(this.virtualClients[domain].userIds[userId]); + const client = this.virtualClients[domain].userIds[userId]; + if (client) { + clientList[userId].push(client); + } }); }); return clientList; } - private checkClientLimit(server: IrcServer) { + private async checkClientLimit(server: IrcServer) { if (server.getMaxClients() === 0) { return; } @@ -259,8 +262,7 @@ export class ClientPool { // find the oldest client to kill. let oldest: BridgedClient|null = null; - Object.keys(this.virtualClients[server.domain].nicks).forEach((nick: string) => { - const client = this.virtualClients[server.domain].nicks[nick]; + for (const client of Object.values(this.virtualClients[server.domain].nicks)) { if (!client) { // possible since undefined/null values can be present from culled entries return; @@ -275,21 +277,20 @@ export class ClientPool { if (client.getLastActionTs() < oldest.getLastActionTs()) { oldest = client; } - }); + } if (!oldest) { return; } // disconnect and remove mappings. this.removeBridgedClient(oldest); - oldest.disconnect("Client limit exceeded: " + server.getMaxClients()).then( - function() { - log.info("Client limit exceeded: Disconnected %s on %s.", - oldest.nick, oldest.server.domain); - }, - function(e: Error) { - log.error("Error when disconnecting %s on server %s: %s", - oldest.nick, oldest.server.domain, JSON.stringify(e)); - }); + const domain = oldest.server.domain; + try { + await oldest.disconnect("limit_reached", `Client limit exceeded: ${server.getMaxClients()}`) + log.info(`Client limit exceeded: Disconnected ${oldest.nick} on ${domain}.`); + } + catch (ex) { + log.error(`Error when disconnecting ${oldest.nick} on server ${domain}: ${JSON.stringify(ex)}`); + } } public countTotalConnections(): number { @@ -327,11 +328,15 @@ export class ClientPool { public getNickUserIdMappingForChannel(server: IrcServer, channel: string): {[nick: string]: string} { const nickUserIdMap: {[nick: string]: string} = {}; const cliSet = this.virtualClients[server.domain].userIds; - Object.keys(cliSet).filter((userId: string) => - cliSet[userId] && cliSet[userId].chanList - && cliSet[userId].chanList.includes(channel) - ).forEach((userId: string) => { - nickUserIdMap[cliSet[userId].nick] = userId; + Object.keys(cliSet).filter((userId: string) => { + if (!userId) { + return false; + } + const cli = cliSet[userId]; + return cli && cli.chanList.includes(channel); + }).forEach((userId: string) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + nickUserIdMap[cliSet[userId]!.nick] = userId; }); // Correctly map the bot too. nickUserIdMap[server.getBotNickname()] = this.ircBridge.getAppServiceUserId(); @@ -349,7 +354,9 @@ export class ClientPool { private removeBridgedClient(bridgedClient: BridgedClient): void { const server = bridgedClient.server; - this.virtualClients[server.domain].userIds[bridgedClient.userId] = undefined; + if (bridgedClient.userId) { + this.virtualClients[server.domain].userIds[bridgedClient.userId] = undefined; + } this.virtualClients[server.domain].nicks[bridgedClient.nick] = undefined; this.virtualClientCounts[server.domain] = this.virtualClientCounts[server.domain] - 1; @@ -381,8 +388,8 @@ export class ClientPool { this.sendConnectionMetric(bridgedClient.server); // remove the pending nick we had set for this user - if (this.virtualClients[bridgedClient.server]) { - delete this.virtualClients[bridgedClient.server].pending[bridgedClient.nick]; + if (this.virtualClients[bridgedClient.server.domain]) { + delete this.virtualClients[bridgedClient.server.domain].pending[bridgedClient.nick]; } if (bridgedClient.disconnectReason === "banned") { @@ -405,6 +412,10 @@ export class ClientPool { const cliConfig = bridgedClient.getClientConfig(); cliConfig.setDesiredNick(bridgedClient.nick); + if (!bridgedClient.matrixUser) { + // no associated matrix user, run away! + return; + } const cli = this.createIrcClient( cliConfig, bridgedClient.matrixUser, bridgedClient.isBot @@ -412,7 +423,7 @@ export class ClientPool { const chanList = bridgedClient.chanList; // remove ref to the disconnected client so it can be GC'd. If we don't do this, // the timeout below holds it in a closure, preventing it from being GC'd. - bridgedClient = undefined; + (bridgedClient as unknown) = undefined; if (chanList.length === 0) { log.info(`Dropping ${cli._id} ${cli.nick} because they are not joined to any channels`); @@ -432,23 +443,16 @@ export class ClientPool { }); } - private reconnectClient(cliChan: ReconnectionItem): void { - const cli = cliChan.cli; - const chanList: string[] = cliChan.chanList; - return cli.connect().then(() => { - log.info( - "<%s> Reconnected %s@%s", cli._id, cli.nick, cli.server.domain - ); - log.info("<%s> Rejoining %s channels", cli._id, chanList.length); - chanList.forEach(function(c: string) { - cli.joinChannel(c); - }); - this.sendConnectionMetric(cli.server); - }, () => { + private async reconnectClient(cliChan: ReconnectionItem) { + try { + await cliChan.cli.reconnect(); + this.sendConnectionMetric(cliChan.cli.server); + } + catch (ex) { log.error( - "<%s> Failed to reconnect %s@%s", cli._id, cli.nick, cli.server.domain + "Failed to reconnect %s@%s", cliChan.cli.nick, cliChan.cli.server.domain ); - }); + } } private onNickChange(bridgedClient: BridgedClient, oldNick: string, newNick: string): void { From 06648d458fa3bf95fae4199a14e96179db811bdf Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:59:25 +0100 Subject: [PATCH 110/350] Tweak ConnectionInstance for BridgedClient types --- src/irc/ConnectionInstance.ts | 82 ++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/src/irc/ConnectionInstance.ts b/src/irc/ConnectionInstance.ts index ad885b1bb..91edda0a8 100644 --- a/src/irc/ConnectionInstance.ts +++ b/src/irc/ConnectionInstance.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +// We have no types for IRC yet. +// eslint-disable-next-line @typescript-eslint/no-var-requires const irc = require("irc"); import * as promiseutil from "../promiseutil"; @@ -25,6 +27,13 @@ import { IrcServer } from "./IrcServer"; const log = logging.get("client-connection"); +export interface IrcError { + command?: string; + args: string[]; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type IrcClient = any; // The time we're willing to wait for a connect callback when connecting to IRC. const CONNECT_TIMEOUT_MS = 30 * 1000; // 30s @@ -61,20 +70,21 @@ function logError(err: Error) { } export interface ConnectionOpts { - localAddress: string; + localAddress?: string; password?: string; realname: string; username: string; nick: string; secure?: { ca?: string; - } + }; } -export type InstanceDisconnectReason = "throttled"|"irc_error"|"net_error"|"timeout"|"raw_error"|"toomanyconns"|"banned"; +export type InstanceDisconnectReason = "throttled"|"irc_error"|"net_error"|"timeout"|"raw_error"| + "toomanyconns"|"banned"|"killed"|"idle"|"limit_reached"; export class ConnectionInstance { - public dead: boolean = false; + public dead = false; private state: "created"|"connecting"|"connected" = "created"; private pingRateTimerId: NodeJS.Timer|null = null; private clientSidePingTimeoutTimerId: NodeJS.Timer|null = null; @@ -88,7 +98,7 @@ export class ConnectionInstance { * @param {string} domain The domain (for logging purposes) * @param {string} nick The nick (for logging purposes) */ - constructor (private readonly client: any, private readonly domain: string, private nick: string) { + constructor (public readonly client: IrcClient, private readonly domain: string, private nick: string) { this.listenForErrors(); this.listenForPings(); this.listenForCTCPVersions(); @@ -130,10 +140,11 @@ export class ConnectionInstance { * @param {string} reason - Reason to reject with. One of: * throttled|irc_error|net_error|timeout|raw_error|toomanyconns|banned */ - public disconnect(reason: InstanceDisconnectReason) { + public disconnect(reason: InstanceDisconnectReason, ircReason?: string) { if (this.dead) { return Bluebird.resolve(); } + ircReason = ircReason || reason; log.info( "disconnect()ing %s@%s - %s", this.nick, this.domain, reason ); @@ -141,7 +152,7 @@ export class ConnectionInstance { return new Bluebird((resolve) => { // close the connection - this.client.disconnect(reason, () => {}); + this.client.disconnect(ircReason, () => {}); // remove timers if (this.pingRateTimerId) { clearTimeout(this.pingRateTimerId); @@ -168,7 +179,7 @@ export class ConnectionInstance { } public addListener(eventName: string, fn: (item: IArguments) => void) { - this.client.addListener(eventName, () => { + this.client.addListener(eventName, (...args: Array) => { if (this.dead) { log.error( "%s@%s RECV a %s event for a dead connection", @@ -177,12 +188,14 @@ export class ConnectionInstance { return; } // do the callback - fn.apply(fn, arguments as any); + // eslint is usually confused about IArguments + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn.apply(fn, args as any); }); } private listenForErrors() { - this.client.addListener("error", (err?: {command?: string}) => { + this.client.addListener("error", (err?: IrcError) => { log.error("Server: %s (%s) Error: %s", this.domain, this.nick, JSON.stringify(err)); // We should disconnect the client for some but not all error codes. This // list is a list of codes which we will NOT disconnect the client for. @@ -222,7 +235,7 @@ export class ConnectionInstance { ); this.disconnect("net_error").catch(logError); }); - this.client.addListener("raw", (msg?: {command?: string, rawCommand: string, args?: string[]}) => { + this.client.addListener("raw", (msg?: {command?: string; rawCommand: string; args?: string[]}) => { if (logging.isVerbose()) { log.debug( "%s@%s: %s", this.nick, this.domain, JSON.stringify(msg) @@ -232,9 +245,9 @@ export class ConnectionInstance { log.error( "%s@%s: %s", this.nick, this.domain, JSON.stringify(msg) ); - var wasThrottled = false; + let wasThrottled = false; if (msg.args) { - var errText = ("" + msg.args[0]) || ""; + let errText = ("" + msg.args[0]) || ""; errText = errText.toLowerCase(); wasThrottled = errText.indexOf("throttl") !== -1; if (wasThrottled) { @@ -259,8 +272,8 @@ export class ConnectionInstance { } } }); - }; - + } + private listenForPings() { // BOTS-65 : A client can get ping timed out and not reconnect. // ------------------------------------------------------------ @@ -290,19 +303,20 @@ export class ConnectionInstance { }); // decorate client.send to refresh the timer const realSend = this.client.send; - this.client.send = (command: string) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.client.send = (...args: unknown[]) => { keepAlivePing(); this.resetPingSendTimer(); // sending a message counts as a ping - realSend.apply(this.client, arguments); + realSend.apply(this.client, args); }; - }; - + } + private listenForCTCPVersions() { this.client.addListener("ctcp-version", (from: string) => { this.client.ctcp(from, 'reply', `VERSION ${CTCP_VERSION}`); }); - }; - + } + private resetPingSendTimer() { // reset the ping rate timer if (this.pingRateTimerId) { @@ -317,7 +331,7 @@ export class ConnectionInstance { // keep doing it. this.resetPingSendTimer(); }, PING_RATE_MS); - } + } /** * Create an IRC client connection and connect to it. @@ -331,11 +345,11 @@ export class ConnectionInstance { * @param {Function} onCreatedCallback Called with the client when created. * @return {Promise} Resolves to an ConnectionInstance or rejects. */ - public static async create (server: IrcServer, opts: ConnectionOpts, onCreatedCallback: (inst: ConnectionInstance) => void) { + public static async create (server: IrcServer, opts: ConnectionOpts, + onCreatedCallback?: (inst: ConnectionInstance) => void): Promise { if (!opts.nick || !server) { throw new Error("Bad inputs. Nick: " + opts.nick); } - onCreatedCallback = onCreatedCallback || function() {}; const connectionOpts = { userName: opts.username, realName: opts.realname, @@ -356,14 +370,16 @@ export class ConnectionInstance { }; // Returns: A promise which resolves to a ConnectionInstance - let retryConnection = () => { - let nodeClient = new irc.Client( + const retryConnection = () => { + const nodeClient = new irc.Client( server.randomDomain(), opts.nick, connectionOpts ); - let inst = new ConnectionInstance( + const inst = new ConnectionInstance( nodeClient, server.domain, opts.nick ); - onCreatedCallback(inst); + if (onCreatedCallback) { + onCreatedCallback(inst); + } return inst.connect(); }; @@ -374,14 +390,12 @@ export class ConnectionInstance { try { if (server.getReconnectIntervalMs() > 0) { // wait until scheduled - let cli = await Scheduler.reschedule( + return (await Scheduler.reschedule( server, retryTimeMs, retryConnection, opts.nick - ); - return cli; + )) as ConnectionInstance; } // Try to connect immediately: we'll wait if we fail. - let cli = await retryConnection(); - return cli; + return await retryConnection(); } catch (err) { connAttempts += 1; @@ -419,4 +433,4 @@ export class ConnectionInstance { } } } -} \ No newline at end of file +} From 5a8ad2f625721b485ffce8eb072d33d54e88333e Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 15:59:40 +0100 Subject: [PATCH 111/350] Typing tweaks to logger --- src/logging.ts | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/logging.ts b/src/logging.ts index d916c4a4f..ae1cc4bc3 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -14,11 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ - -/* - * This module provides python-like logging capabilities using winston. - */ - import winston, { TransportInstance, LeveledLogMethod, LoggerInstance } from "winston"; import "winston-daily-rotate-file"; import { WriteStream } from "fs"; @@ -72,7 +67,7 @@ export function formatterFn(opts: FormatterFnOpts) { const makeTransports = function() { - let transports = []; + const transports = []; if (loggerConfig.toConsole) { transports.push(new (winston.transports.Console)({ json: false, @@ -146,7 +141,7 @@ export function get(nameOfLogger: string) { if (loggers[nameOfLogger]) { return loggers[nameOfLogger]; } - let logger = createLogger(nameOfLogger); + const logger = createLogger(nameOfLogger); loggers[nameOfLogger] = logger; const ircLogger = { logErr: (e: Error) => { @@ -175,9 +170,9 @@ export function configure(opts: LoggerConfig) { // with the default config, which is now being overwritten by this // configure() call. Object.keys(loggers).forEach(function(loggerName) { - let existingLogger = loggers[loggerName]; + const existingLogger = loggers[loggerName]; // remove each individual transport - let transportNames = ["logfile", "console", "errorfile"]; + const transportNames = ["logfile", "console", "errorfile"]; transportNames.forEach(function(tname) { if (existingLogger.transports[tname]) { existingLogger.remove(tname); @@ -194,9 +189,11 @@ export function isVerbose() { return loggerConfig.verbose; } +// We use any a lot here to avoid having to deal with IArguments inflexibity +/* eslint-disable @typescript-eslint/no-explicit-any */ export function newRequestLogger(baseLogger: LoggerInstance, requestId: string, isFromIrc: boolean) { - const decorate = function(fn: LeveledLogMethod, args: IArguments ) { - let newArgs: Array = []; + const decorate = function(fn: LeveledLogMethod, args: any[] ) { + const newArgs: Array = []; // don't slice this; screws v8 optimisations apparently for (let i = 0; i < args.length; i++) { newArgs.push(args[i]); @@ -206,17 +203,16 @@ export function newRequestLogger(baseLogger: LoggerInstance, requestId: string, reqId: requestId, dir: (isFromIrc ? "[I->M] " : "[M->I] ") }; - // Typescript doesn't like us mangling args like this, but we have to. - // @ts-ignore - fn.apply(baseLogger, newArgs); + fn.apply(baseLogger, newArgs as any); }; return { - debug: function() { decorate(baseLogger.debug, arguments); }, - info: function() { decorate(baseLogger.info, arguments); }, - warn: function() { decorate(baseLogger.warn, arguments); }, - error: function() { decorate(baseLogger.error, arguments); }, + debug: (...args: any[]) => { decorate(baseLogger.debug, args); }, + info: (...args: any[]) => { decorate(baseLogger.info, args); }, + warn: (...args: any[]) => { decorate(baseLogger.warn, args); }, + error: (...args: any[]) => { decorate(baseLogger.error, args); }, }; } +/* eslint-enable @typescript-eslint/no-explicit-any */ export function setUncaughtExceptionLogger(exceptionLogger: LoggerInstance) { process.on("uncaughtException", function(e) { @@ -240,7 +236,7 @@ export function setUncaughtExceptionLogger(exceptionLogger: LoggerInstance) { // few lines before quitting, which I suspect is due to it not flushing. // Since we know we're going to die at this point, log something else // and forcibly flush all the transports before exiting. - exceptionLogger.error("Terminating (exitcode=1)", function(err: Error) { + exceptionLogger.error("Terminating (exitcode=1)", function() { let numFlushes = 0; let numFlushed = 0; Object.keys(exceptionLogger.transports).forEach(function(k) { From 52580730cc257b25a17c740851a51c0e2f68b9e1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 16:00:02 +0100 Subject: [PATCH 112/350] Cleanup scheduler --- src/irc/Scheduler.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/irc/Scheduler.ts b/src/irc/Scheduler.ts index 951878e70..abbf74ddf 100644 --- a/src/irc/Scheduler.ts +++ b/src/irc/Scheduler.ts @@ -22,8 +22,8 @@ import { IrcServer } from "./IrcServer"; const log = getLogger("scheduler"); interface QueueItem { - fn: () => Promise, - addedDelayMs: number, + fn: () => Promise; + addedDelayMs: number; } // Maps domain => Queue @@ -54,10 +54,14 @@ export default { // Returns a promise that will be resolved when retryConnection returns a promise that // resolves, in other words, when the connection is made. The promise will reject if the // promise returned from retryConnection is rejected. - reschedule: Bluebird.coroutine(function*(server: IrcServer, addedDelayMs: number, retryConnection: () => Promise, nick: string) { - var q = getQueue(server); + reschedule: Bluebird.coroutine(function*( + server: IrcServer, + addedDelayMs: number, + retryConnection: () => Promise, + nick: string) { + const q = getQueue(server); - var promise = q.enqueue( + const promise = q.enqueue( `Scheduler.reschedule ${server.domain} ${nick}`, { fn: retryConnection, @@ -79,9 +83,9 @@ export default { // Reject all queued promises killAll: function () { - let queueKeys = Object.keys(queues); - for (var i = 0; i < queueKeys.length; i++) { - var q = queues[queueKeys[i]]; + const queueKeys = Object.keys(queues); + for (let i = 0; i < queueKeys.length; i++) { + const q = queues[queueKeys[i]]; q.killAll(); } } From ce550f35d96d63f857b44bf4e26e9309c37966e8 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 16:32:05 +0100 Subject: [PATCH 113/350] Fix test --- src/DebugApi.ts | 2 +- src/datastore/postgres/PgDataStore.ts | 4 ++-- src/irc/BridgedClient.ts | 1 + src/irc/ClientPool.ts | 8 ++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/DebugApi.ts b/src/DebugApi.ts index 3bd24e24e..5f55d92e5 100644 --- a/src/DebugApi.ts +++ b/src/DebugApi.ts @@ -43,7 +43,7 @@ export class DebugApi { try { this.onRequest(req, res); } - catch (err) { + catch (err) { if (!res.finished) { res.end(); } diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index 08ecc3af2..3d940cefe 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -516,7 +516,7 @@ export class PgDataStore implements DataStore { currentVersion++; await this.updateSchemaVersion(currentVersion); } - catch (ex) { + catch (ex) { log.warn(`Failed to run schema v${currentVersion + 1}:`, ex); throw Error("Failed to update database schema"); } @@ -546,7 +546,7 @@ export class PgDataStore implements DataStore { const { rows } = await this.pgPool.query("SELECT version FROM SCHEMA"); return rows[0].version; } - catch (ex) { + catch (ex) { if (ex.code === "42P01") { // undefined_table log.warn("Schema table could not be found"); return 0; diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts index 64e4aa82e..56423e835 100644 --- a/src/irc/BridgedClient.ts +++ b/src/irc/BridgedClient.ts @@ -284,6 +284,7 @@ export class BridgedClient extends EventEmitter { } public async reconnect() { + await this.connect(); this.log.info( "Reconnected %s@%s", this.nick, this.server.domain ); diff --git a/src/irc/ClientPool.ts b/src/irc/ClientPool.ts index 57e955d16..5cd44c59c 100644 --- a/src/irc/ClientPool.ts +++ b/src/irc/ClientPool.ts @@ -265,14 +265,14 @@ export class ClientPool { for (const client of Object.values(this.virtualClients[server.domain].nicks)) { if (!client) { // possible since undefined/null values can be present from culled entries - return; + continue; } if (client.isBot) { - return; // don't ever kick the bot off. + continue; // don't ever kick the bot off. } if (oldest === null) { oldest = client; - return; + continue; } if (client.getLastActionTs() < oldest.getLastActionTs()) { oldest = client; @@ -448,7 +448,7 @@ export class ClientPool { await cliChan.cli.reconnect(); this.sendConnectionMetric(cliChan.cli.server); } - catch (ex) { + catch (ex) { log.error( "Failed to reconnect %s@%s", cliChan.cli.nick, cliChan.cli.server.domain ); From a1fea4d4da37b05343d94fecffc4fe17901fde24 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 16:55:02 +0100 Subject: [PATCH 114/350] Convert IdentGenerator to Typescript --- spec/unit/IdentGenerator.spec.js | 77 +++++------ src/bridge/IrcBridge.js | 2 +- src/irc/BridgedClient.ts | 6 +- src/irc/ConnectionInstance.ts | 2 +- src/irc/IdentGenerator.js | 224 ------------------------------ src/irc/IdentGenerator.ts | 231 +++++++++++++++++++++++++++++++ src/main.js | 2 +- 7 files changed, 271 insertions(+), 273 deletions(-) delete mode 100644 src/irc/IdentGenerator.js create mode 100644 src/irc/IdentGenerator.ts diff --git a/spec/unit/IdentGenerator.spec.js b/spec/unit/IdentGenerator.spec.js index 2fe643bea..91add7279 100644 --- a/spec/unit/IdentGenerator.spec.js +++ b/spec/unit/IdentGenerator.spec.js @@ -1,6 +1,6 @@ "use strict"; const Promise = require("bluebird"); -const IdentGenerator = require("../../lib/irc/IdentGenerator.js"); +const { IdentGenerator } = require("../../lib/irc/IdentGenerator.js"); describe("Username generation", function() { var identGenerator; @@ -49,71 +49,62 @@ describe("Username generation", function() { IdentGenerator.MAX_USER_NAME_LENGTH = 8; }); - it("should attempt to truncate the user ID on a long user ID", function(done) { + it("should attempt to truncate the user ID on a long user ID", async function() { var userId = "@myreallylonguseridhere:localhost"; var uname = "myreally"; - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(info.username).toEqual(uname); - done(); - }); + const info = await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + expect(info.username).toEqual(uname); }); - it("should start with '_1' on an occupied user ID", function(done) { - var userId = "@myreallylonguseridhere:localhost"; - var uname = "myreal_1"; + it("should start with '_1' on an occupied user ID", async function() { + const userId = "@myreallylonguseridhere:localhost"; + const uname = "myreal_1"; existingUsernames.myreally = "@someone:else"; - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(info.username).toEqual(uname); - done(); - }); + const info = await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + expect(info.username).toEqual(uname); }); - it("should loop from '_9' to '_10' and keep the same total length", function(done) { - var userId = "@myreallylonguseridhere:localhost"; - var uname = "myrea_10"; + it("should loop from '_9' to '_10' and keep the same total length", async function() { + const userId = "@myreallylonguseridhere:localhost"; + const uname = "myrea_10"; existingUsernames.myreally = "@someone:else"; - for (var i = 1; i < 10; i++) { + for (let i = 1; i < 10; i++) { existingUsernames["myreal_" + i] = "@someone:else"; } - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(info.username).toEqual(uname); - done(); - }); + const info = await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + expect(info.username).toEqual(uname); }); - it("should loop from '_1' to '_2' and keep the same total length", function(done) { - var userId = "@myreallylonguseridhere:localhost"; - var uname = "myreal_2"; + it("should loop from '_1' to '_2' and keep the same total length", async function() { + const userId = "@myreallylonguseridhere:localhost"; + const uname = "myreal_2"; existingUsernames = { myreally: "@someone:else", myreal_1: "@someone:else" }; - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(info.username).toEqual(uname); - done(); - }); + const info = await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + expect(info.username).toEqual(uname); }); - it("should eventually give up trying usernames", function(done) { + it("should eventually give up trying usernames", async function() { IdentGenerator.MAX_USER_NAME_LENGTH = 3; storeMock.getMatrixUserByUsername = function() { return Promise.resolve({getId: function() { return "@someone:else"} }); }; - var userId = "@myreallylonguseridhere:localhost"; - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(true).toBe(false, "Promise was unexpectedly resolved."); - done(); - }, function(err) { - done(); - }); + const userId = "@myreallylonguseridhere:localhost"; + try { + await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + } + catch (ex) { + return; + } + throw Error("Promise was unexpectedly resolved"); }); - it("should prefix 'M' onto usernames which don't begin with A-z", function(done) { - var userId = "@-myname:localhost"; - var uname = "M-myname"; - identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)).done(function(info) { - expect(info.username).toEqual(uname); - done(); - }); + it("should prefix 'M' onto usernames which don't begin with A-z", async function() { + const userId = "@-myname:localhost"; + const uname = "M-myname"; + const info = await identGenerator.getIrcNames(ircClientConfig, mkMatrixUser(userId)); + expect(info.username).toEqual(uname); }); }); diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 6cad62ecf..3fc0623e8 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -7,7 +7,7 @@ var promiseutil = require("../promiseutil"); var IrcHandler = require("./IrcHandler.js"); var MatrixHandler = require("./MatrixHandler.js"); var MemberListSyncer = require("./MemberListSyncer.js"); -var IdentGenerator = require("../irc/IdentGenerator.js"); +const { IdentGenerator } = require("../irc/IdentGenerator.js"); var Ipv6Generator = require("../irc/Ipv6Generator.js"); const { IrcServer } = require("../irc/IrcServer.js"); const { ClientPool } = require("../irc/ClientPool"); diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts index 56423e835..fce18bc91 100644 --- a/src/irc/BridgedClient.ts +++ b/src/irc/BridgedClient.ts @@ -26,6 +26,7 @@ import { IrcClientConfig } from "../models/IrcClientConfig"; import { MatrixUser } from "matrix-appservice-bridge"; import { LoggerInstance } from "winston"; import { IrcAction } from "../models/IrcAction"; +import { IdentGenerator } from "./IdentGenerator"; const log = getLogger("BridgedClient"); @@ -36,7 +37,6 @@ const NICK_DELAY_TIMER_MS = 10 * 1000; // 10s // All of these are not defined yet. /* eslint-disable @typescript-eslint/no-explicit-any */ type EventBroker = any; -type IdentGenerator = any; type Ipv6Generator = any; type IrcClient = EventEmitter|any; /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -680,7 +680,7 @@ export class BridgedClient extends EventEmitter { return this.lastActionTs; } - private onConnectionCreated(connInst: ConnectionInstance, nameInfo: {username: string}) { + private onConnectionCreated(connInst: ConnectionInstance, nameInfo: {username?: string}) { // listen for a connect event which is done when the TCP connection is // established and set ident info (this is different to the connect() callback // in node-irc which actually fires on a registered event..) @@ -689,7 +689,7 @@ export class BridgedClient extends EventEmitter { if (connInst.client.conn && connInst.client.conn.localPort) { localPort = connInst.client.conn.localPort; } - if (localPort > 0) { + if (localPort > 0 && nameInfo.username) { Ident.setMapping(nameInfo.username, localPort); } }); diff --git a/src/irc/ConnectionInstance.ts b/src/irc/ConnectionInstance.ts index 91edda0a8..5cfc48bda 100644 --- a/src/irc/ConnectionInstance.ts +++ b/src/irc/ConnectionInstance.ts @@ -73,7 +73,7 @@ export interface ConnectionOpts { localAddress?: string; password?: string; realname: string; - username: string; + username?: string; nick: string; secure?: { ca?: string; diff --git a/src/irc/IdentGenerator.js b/src/irc/IdentGenerator.js deleted file mode 100644 index 8c06bd402..000000000 --- a/src/irc/IdentGenerator.js +++ /dev/null @@ -1,224 +0,0 @@ -/*eslint no-invalid-this: 0 no-constant-condition: 0 */ -"use strict"; -const Promise = require("bluebird"); -const { Queue } = require("../util/Queue"); -const log = require("../logging").get("IdentGenerator"); - -function IdentGenerator(store) { - // Queue of ident generation requests. - // We need to queue them because otherwise 2 clashing user_ids could be assigned - // the same ident value (won't be in the database yet) - this.queue = new Queue(this._process.bind(this)); - this.dataStore = store; -} - -// debugging: util.inspect() -IdentGenerator.prototype.inspect = function(depth) { - return "IdentGenerator queue length=" + - (this.queue._queue ? - this.queue._queue.length : -1); -} - - -/** - * Get the IRC name info for this user. - * @param {IrcClientConfig} clientConfig IRC client configuration info. - * @param {MatrixUser} matrixUser Optional. The matrix user. - * @return {Promise} Resolves to { - * username: 'username_to_use', - * realname: 'realname_to_use' - * } - */ -IdentGenerator.prototype.getIrcNames = Promise.coroutine(function*(ircClientConfig, matrixUser) { - var info = { - username: null, - realname: (matrixUser ? - sanitiseRealname(matrixUser.getId()) : - sanitiseRealname(ircClientConfig.getUsername()) - ).substring( - 0, IdentGenerator.MAX_REAL_NAME_LENGTH - ) - }; - if (matrixUser) { - if (ircClientConfig.getUsername()) { - log.debug( - "Using cached ident username %s for %s on %s", - ircClientConfig.getUsername(), matrixUser.getId(), ircClientConfig.getDomain() - ); - info.username = sanitiseUsername(ircClientConfig.getUsername()); - info.username = info.username.substring( - 0, IdentGenerator.MAX_USER_NAME_LENGTH - ); - } - else { - try { - log.debug( - "Pushing username generation request for %s on %s to the queue...", - matrixUser.getId(), ircClientConfig.getDomain() - ); - let uname = yield this.queue.enqueue(matrixUser.getId(), { - matrixUser: matrixUser, - ircClientConfig: ircClientConfig - }); - info.username = uname; - } - catch (err) { - log.error( - "Failed to generate ident username for %s on %s", - matrixUser.getId(), ircClientConfig.getDomain() - ); - log.error(err.stack); - throw err; - } - } - } - else { - info.username = sanitiseUsername( - ircClientConfig.getUsername() // the bridge won't have a matrix user - ); - } - return info; -}); - -IdentGenerator.prototype._process = Promise.coroutine(function*(item) { - var matrixUser = item.matrixUser; - var ircClientConfig = item.ircClientConfig; - var configDomain = ircClientConfig.getDomain(); - - log.debug( - "Generating username for %s on %s", matrixUser.getId(), configDomain - ); - let uname = yield this._generateIdentUsername( - configDomain, matrixUser.getId() - ); - let existingConfig = yield this.dataStore.getIrcClientConfig( - matrixUser.getId(), configDomain - ); - let config = existingConfig ? existingConfig : ircClientConfig; - config.setUsername(uname); - - // persist to db here before releasing the lock on this request. - yield this.dataStore.storeIrcClientConfig(config); - return config.getUsername(); -}); - -/** - * Generate a new IRC username for the given Matrix user on the given server. - * @param {string} domain The IRC server domain - * @param {string} userId The matrix user being bridged - * @return {Promise} resolves to the username {string}. - */ -IdentGenerator.prototype._generateIdentUsername = Promise.coroutine(function*(domain, userId) { - // @foobar££stuff:domain.com => foobar__stuff_domain_com - var uname = sanitiseUsername(userId.substring(1)); - if (uname < IdentGenerator.MAX_USER_NAME_LENGTH) { // bwahaha not likely. - return uname; - } - uname = uname.substring(0, IdentGenerator.MAX_USER_NAME_LENGTH); - /* LONGNAM~1 ing algorithm: - * foobar => foob~1 => foob~2 => ... => foob~9 => foo~10 => foo~11 => ... - * f~9999 => FAIL. - * - * Ideal data structure (Tries): TODO - * (each level from the root node increases digit count by 1) - * .---[f]---. Translation: - * 123[o] [a]743 Up to fo~123 is taken - * | Up to fa~743 is taken - * 34[o] Up to foo~34 is taken - * | Up to foot~9 is taken (re-search as can't increment) - * 9[t] - * - * while not_free(uname): - * if ~ not in uname: - * uname = uname[0:-2] + "~1" // foobar => foob~1 - * continue - * [name, num] = uname.split(~) // foob~9 => ['foob', '9'] - * old_digits_len = len(str(num)) // '9' => 1 - * num += 1 - * new_digits_len = len(str(num)) // '10' => 2 - * if new_digits_len > old_digits_len: - * uname = name[:-1] + "~" + num // foob,10 => foo~10 - * else: - * uname = name + "~" + num // foob,8 => foob~8 - * - * return uname - */ - var delim = "_"; - function modifyUsername() { - if (uname.indexOf(delim) === -1) { - uname = uname.substring(0, uname.length - 2) + delim + "1"; - return true; - } - var segments = uname.split(delim); - var oldLen = segments[1].length; - var num = parseInt(segments[1]) + 1; - if (("" + num).length > oldLen) { - uname = segments[0].substring(0, segments[0].length - 1) + delim + num; - } - else { - uname = segments[0] + delim + num; - } - return uname.indexOf(delim) !== 0; // break out if '~10000' - } - - // TODO: This isn't efficient currently; since this will be called worst - // case 10^[num chars in string] => 10^10 - // We should instead be querying to extract the max occupied number for - // that char string (which is worst case [num chars in string]), e.g. - // fooba => 9, foob => 99, foo => 999, fo => 4523 = fo~4524 - while (true) { - let usr = yield this.dataStore.getMatrixUserByUsername(domain, uname); - if (usr && usr.getId() !== userId) { // occupied username! - if (!modifyUsername()) { - throw new Error("Ran out of entries: " + uname); - } - } - else { - if (!usr) { - log.info( - "Generated ident username %s for %s on %s", - uname, userId, domain - ); - } - else { - log.info( - "Returning cached ident username %s for %s on %s", - uname, userId, domain - ); - } - break; - } - } - return uname; -}); - -function sanitiseUsername(username, replacementChar) { - replacementChar = replacementChar || ""; // default remove chars - username = username.toLowerCase(); - // strip illegal chars according to RFC 1459 Sect 2.3.1 - // (technically it's any ascii for but meh) - // also strip '_' since we use that as the delimiter - username = username.replace(/[^A-Za-z0-9\]\[\^\\\{\}\-`]/g, replacementChar); - // Whilst the RFC doesn't say you can't have special characters eg ("-") as the - // first character of a USERNAME, empirically Freenode rejects connections - // stating "Invalid username". Having "-" is valid, so long as it isn't the first. - // Prefix usernames with "M" if they start with a special character. - if (/^[^A-Za-z]/.test(username)) { - return "M" + username; - } - return username; -} - -function sanitiseRealname(realname) { - // real name can be any old ASCII - return realname.replace(/[^\x00-\x7F]/g, ""); -} - -// The max length of in USER commands -IdentGenerator.MAX_REAL_NAME_LENGTH = 48; - -// The max length of in USER commands -IdentGenerator.MAX_USER_NAME_LENGTH = 10; - - -module.exports = IdentGenerator; diff --git a/src/irc/IdentGenerator.ts b/src/irc/IdentGenerator.ts new file mode 100644 index 000000000..0a912ed0d --- /dev/null +++ b/src/irc/IdentGenerator.ts @@ -0,0 +1,231 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Queue } from "../util/Queue"; +import { getLogger } from "../logging"; +import { DataStore } from "../datastore/DataStore"; +import { MatrixUser } from "matrix-appservice-bridge"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +const log = getLogger("IdentGenerator"); + +export class IdentGenerator { + // The max length of in USER commands + private static readonly MAX_REAL_NAME_LENGTH = 48; + // The max length of in USER commands + private static readonly MAX_USER_NAME_LENGTH = 10; + + private queue: Queue; + constructor (private readonly dataStore: DataStore) { + // Queue of ident generation requests. + // We need to queue them because otherwise 2 clashing user_ids could be assigned + // the same ident value (won't be in the database yet) + this.queue = new Queue((item: unknown) => { + const {matrixUser, ircClientConfig} = item as { matrixUser: MatrixUser, ircClientConfig: IrcClientConfig}; + return this.process(matrixUser, ircClientConfig); + }); + } + + /** + * Get the IRC name info for this user. + * @param {IrcClientConfig} clientConfig IRC client configuration info. + * @param {MatrixUser} matrixUser Optional. The matrix user. + * @return {Promise} Resolves to { + * username: 'username_to_use', + * realname: 'realname_to_use' + * } + */ + public async getIrcNames(ircClientConfig: IrcClientConfig, matrixUser?: MatrixUser) { + const username = ircClientConfig.getUsername(); + const info: {username?: string, realname: string} = { + username: undefined, + realname: (matrixUser ? + IdentGenerator.sanitiseRealname(matrixUser.getId()) : + IdentGenerator.sanitiseRealname(username || "") + ).substring( + 0, IdentGenerator.MAX_REAL_NAME_LENGTH + ), + }; + if (matrixUser) { + if (username) { + log.debug( + "Using cached ident username %s for %s on %s", + ircClientConfig.getUsername(), matrixUser.getId(), ircClientConfig.getDomain() + ); + info.username = IdentGenerator.sanitiseUsername(username); + info.username = info.username.substring( + 0, IdentGenerator.MAX_USER_NAME_LENGTH + ); + } + else { + try { + log.debug( + "Pushing username generation request for %s on %s to the queue...", + matrixUser.getId(), ircClientConfig.getDomain() + ) + const uname = await this.queue.enqueue(matrixUser.getId(), { + matrixUser: matrixUser, + ircClientConfig: ircClientConfig + }) + info.username = uname as string; + } + catch (err) { + log.error( + "Failed to generate ident username for %s on %s", + matrixUser.getId(), ircClientConfig.getDomain() + ) + log.error(err.stack); + throw err; + } + } + } + else if (username) { + info.username = IdentGenerator.sanitiseUsername( + username // the bridge won't have a matrix user + ) + } + return info; + } + + // debugging: util.inspect() + public inspect() { + return `IdentGenerator queue length=${this.queue.size}` + } + + private async process (matrixUser: MatrixUser, ircClientConfig: IrcClientConfig) { + const configDomain = ircClientConfig.getDomain(); + log.debug("Generating username for %s on %s", matrixUser.getId(), configDomain); + const uname = await this.generateIdentUsername(configDomain, matrixUser.getId()); + const existingConfig = await this.dataStore.getIrcClientConfig(matrixUser.getId(), configDomain); + const config = existingConfig ? existingConfig : ircClientConfig; + config.setUsername(uname); + + // persist to db here before releasing the lock on this request. + await this.dataStore.storeIrcClientConfig(config); + return config.getUsername(); + } + + /** + * Generate a new IRC username for the given Matrix user on the given server. + * @param {string} domain The IRC server domain + * @param {string} userId The matrix user being bridged + * @return {Promise} resolves to the username {string}. + */ + private async generateIdentUsername(domain: string, userId: string) { + // @foobar££stuff:domain.com => foobar__stuff_domain_com + let uname = IdentGenerator.sanitiseUsername(userId.substring(1)); + if (uname.length < IdentGenerator.MAX_USER_NAME_LENGTH) { // bwahaha not likely. + return uname; + } + uname = uname.substring(0, IdentGenerator.MAX_USER_NAME_LENGTH); + /* LONGNAM~1 ing algorithm: + * foobar => foob~1 => foob~2 => ... => foob~9 => foo~10 => foo~11 => ... + * f~9999 => FAIL. + * + * Ideal data structure (Tries): TODO + * (each level from the root node increases digit count by 1) + * .---[f]---. Translation: + * 123[o] [a]743 Up to fo~123 is taken + * | Up to fa~743 is taken + * 34[o] Up to foo~34 is taken + * | Up to foot~9 is taken (re-search as can't increment) + * 9[t] + * + * while not_free(uname): + * if ~ not in uname: + * uname = uname[0:-2] + "~1" // foobar => foob~1 + * continue + * [name, num] = uname.split(~) // foob~9 => ['foob', '9'] + * old_digits_len = len(str(num)) // '9' => 1 + * num += 1 + * new_digits_len = len(str(num)) // '10' => 2 + * if new_digits_len > old_digits_len: + * uname = name[:-1] + "~" + num // foob,10 => foo~10 + * else: + * uname = name + "~" + num // foob,8 => foob~8 + * + * return uname + */ + var delim = "_"; + const modifyUsername = () => { + if (uname.indexOf(delim) === -1) { + uname = uname.substring(0, uname.length - 2) + delim + "1"; + return true; + } + const segments = uname.split(delim); + const oldLen = segments[1].length; + const num = parseInt(segments[1]) + 1; + if (("" + num).length > oldLen) { + uname = segments[0].substring(0, segments[0].length - 1) + delim + num; + } + else { + uname = segments[0] + delim + num; + } + return uname.indexOf(delim) !== 0; // break out if '~10000' + } + + // TODO: This isn't efficient currently; since this will be called worst + // case 10^[num chars in string] => 10^10 + // We should instead be querying to extract the max occupied number for + // that char string (which is worst case [num chars in string]), e.g. + // fooba => 9, foob => 99, foo => 999, fo => 4523 = fo~4524 + while (true) { + const usr = await this.dataStore.getMatrixUserByUsername(domain, uname); + if (usr && usr.getId() !== userId) { // occupied username! + if (!modifyUsername()) { + throw new Error("Ran out of entries: " + uname); + } + } + else { + if (!usr) { + log.info( + "Generated ident username %s for %s on %s", + uname, userId, domain + ); + } + else { + log.info( + "Returning cached ident username %s for %s on %s", + uname, userId, domain + ); + } + break; + } + } + return uname; + } + + private static sanitiseUsername(username: string, replacementChar: string = "") { + username = username.toLowerCase(); + // strip illegal chars according to RFC 1459 Sect 2.3.1 + // (technically it's any ascii for but meh) + // also strip '_' since we use that as the delimiter + username = username.replace(/[^A-Za-z0-9\]\[\^\\\{\}\-`]/g, replacementChar); + // Whilst the RFC doesn't say you can't have special characters eg ("-") as the + // first character of a USERNAME, empirically Freenode rejects connections + // stating "Invalid username". Having "-" is valid, so long as it isn't the first. + // Prefix usernames with "M" if they start with a special character. + if (/^[^A-Za-z]/.test(username)) { + return "M" + username; + } + return username; + } + + private static sanitiseRealname(realname: string) { + // real name can be any old ASCII + return realname.replace(/[^\x00-\x7F]/g, ""); + } +} diff --git a/src/main.js b/src/main.js index f45db70b2..5c5cdbaf3 100644 --- a/src/main.js +++ b/src/main.js @@ -9,7 +9,7 @@ const UserBridgeStore = require("matrix-appservice-bridge").UserBridgeStore; const IrcBridge = require("./bridge/IrcBridge.js"); const { IrcServer } = require("./irc/IrcServer.js"); const stats = require("./config/stats"); -const ident = require("./irc/Ident"); +const ident = require("./irc/Ident").default; const logging = require("./logging"); const log = logging.get("main"); From dcd3bc17e921d7d3ec26a4a31143d8d786b7717c Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 17:15:40 +0100 Subject: [PATCH 115/350] Convert Ipv6Generator to Typescript --- src/bridge/IrcBridge.js | 2 +- src/irc/BridgedClient.ts | 7 ++-- src/irc/Ipv6Generator.js | 85 ------------------------------------- src/irc/Ipv6Generator.ts | 90 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 89 deletions(-) delete mode 100644 src/irc/Ipv6Generator.js create mode 100644 src/irc/Ipv6Generator.ts diff --git a/src/bridge/IrcBridge.js b/src/bridge/IrcBridge.js index 3fc0623e8..813f955d0 100644 --- a/src/bridge/IrcBridge.js +++ b/src/bridge/IrcBridge.js @@ -8,7 +8,7 @@ var IrcHandler = require("./IrcHandler.js"); var MatrixHandler = require("./MatrixHandler.js"); var MemberListSyncer = require("./MemberListSyncer.js"); const { IdentGenerator } = require("../irc/IdentGenerator.js"); -var Ipv6Generator = require("../irc/Ipv6Generator.js"); +const { Ipv6Generator } = require("../irc/Ipv6Generator.js"); const { IrcServer } = require("../irc/IrcServer.js"); const { ClientPool } = require("../irc/ClientPool"); var IrcEventBroker = require("../irc/IrcEventBroker"); diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts index fce18bc91..82dd8f6cd 100644 --- a/src/irc/BridgedClient.ts +++ b/src/irc/BridgedClient.ts @@ -27,6 +27,7 @@ import { MatrixUser } from "matrix-appservice-bridge"; import { LoggerInstance } from "winston"; import { IrcAction } from "../models/IrcAction"; import { IdentGenerator } from "./IdentGenerator"; +import { Ipv6Generator } from "./Ipv6Generator"; const log = getLogger("BridgedClient"); @@ -37,7 +38,6 @@ const NICK_DELAY_TIMER_MS = 10 * 1000; // 10s // All of these are not defined yet. /* eslint-disable @typescript-eslint/no-explicit-any */ type EventBroker = any; -type Ipv6Generator = any; type IrcClient = EventEmitter|any; /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -202,10 +202,11 @@ export class BridgedClient extends EventEmitter { const nameInfo = await this.identGenerator.getIrcNames( this.clientConfig, this.matrixUser ); - if (this.server.getIpv6Prefix()) { + const ipv6Prefix = this.server.getIpv6Prefix(); + if (ipv6Prefix) { // side-effects setting the IPv6 address on the client config await this.ipv6Generator.generate( - this.server.getIpv6Prefix(), this.clientConfig + ipv6Prefix, this.clientConfig ); } this.log.info( diff --git a/src/irc/Ipv6Generator.js b/src/irc/Ipv6Generator.js deleted file mode 100644 index c86b96014..000000000 --- a/src/irc/Ipv6Generator.js +++ /dev/null @@ -1,85 +0,0 @@ -/*eslint no-invalid-this: 0 */ -"use strict"; -const Promise = require("bluebird"); -const { Queue } = require("../util/Queue"); -const log = require("../logging").get("Ipv6Generator"); - -function Ipv6Generator(store) { - // Queue of ipv6 generation requests. - // We need to queue them because otherwise 2 clashing user_ids could be assigned - // the same ipv6 value (won't be in the database yet) - this._queue = new Queue(this._process.bind(this)); - this._dataStore = store; - this._counter = -1; -} - -// debugging: util.inspect() -Ipv6Generator.prototype.inspect = function(depth) { - return "IPv6Counter=" + this._counter + - ",Queue.length=" + (this._queue._queue ? - this._queue._queue.length : -1); -} - -/** - * Generate a new IPv6 address for the given IRC client config. - * @param {string} prefix The IPv6 prefix to use. - * @param {IrcClientConfig} ircClientConfig The config to set the address on. - * @return {Promise} Resolves to the IPv6 address generated; the IPv6 address will - * already be set on the given config. - */ -Ipv6Generator.prototype.generate = Promise.coroutine(function*(prefix, ircClientConfig) { - if (ircClientConfig.getIpv6Address()) { - log.info( - "Using existing IPv6 address %s for %s", - ircClientConfig.getIpv6Address(), - ircClientConfig.getUserId() - ); - return ircClientConfig.getIpv6Address(); - } - if (this._counter === -1) { - log.info("Retrieving counter..."); - this._counter = yield this._dataStore.getIpv6Counter(); - } - - // the bot user will not have a user ID - let id = ircClientConfig.getUserId() || ircClientConfig.getUsername(); - log.info("Enqueueing IPv6 generation request for %s", id); - yield this._queue.enqueue(id, { - prefix: prefix, - ircClientConfig: ircClientConfig - }); - return undefined; -}); - -Ipv6Generator.prototype._process = Promise.coroutine(function*(item) { - this._counter += 1; - - // insert : every 4 characters from the end of the string going to the start - // e.g. 1a2b3c4d5e6 => 1a2:b3c4:d5e6 - let suffix = this._counter.toString(16); - suffix = suffix.replace(/\B(?=(.{4})+(?!.))/g, ':'); - let address = item.prefix + suffix; - - let config = item.ircClientConfig; - config.setIpv6Address(address); - - // we only want to persist the IPv6 address for real matrix users - if (item.ircClientConfig.getUserId()) { - let existingConfig = yield this._dataStore.getIrcClientConfig( - item.ircClientConfig.getUserId(), item.ircClientConfig.getDomain() - ); - if (existingConfig) { - config = existingConfig; - config.setIpv6Address(address); - } - log.info("Generated new IPv6 address %s for %s", address, config.getUserId()); - // persist to db here before releasing the lock on this request. - yield this._dataStore.storeIrcClientConfig(config); - } - - yield this._dataStore.setIpv6Counter(this._counter); - - return config.getIpv6Address(); -}); - -module.exports = Ipv6Generator; diff --git a/src/irc/Ipv6Generator.ts b/src/irc/Ipv6Generator.ts new file mode 100644 index 000000000..25390dd7e --- /dev/null +++ b/src/irc/Ipv6Generator.ts @@ -0,0 +1,90 @@ + +import Bluebird from "bluebird"; +import { Queue } from "../util/Queue"; +import { getLogger } from "../logging"; +import { DataStore } from "../datastore/DataStore"; +import { IrcClientConfig } from "../models/IrcClientConfig"; + +const log = getLogger("Ipv6Generator"); + +export class Ipv6Generator { + private counter: number = -1; + private queue: Queue; + constructor (private readonly dataStore: DataStore) { + // Queue of ipv6 generation requests. + // We need to queue them because otherwise 2 clashing user_ids could be assigned + // the same ipv6 value (won't be in the database yet) + this.queue = new Queue((item) => { + const processItem = item as {prefix: string, ircClientConfig: IrcClientConfig}; + return this.process(processItem.prefix, processItem.ircClientConfig); + }); + } + + // debugging: util.inspect() + public inspect () { + return `IPv6Counter=${this.counter},Queue.length=${this.queue.size}`; + } + + /** + * Generate a new IPv6 address for the given IRC client config. + * @param {string} prefix The IPv6 prefix to use. + * @param {IrcClientConfig} ircClientConfig The config to set the address on. + * @return {Promise} Resolves to the IPv6 address generated; the IPv6 address will + * already be set on the given config. + */ + public async generate (prefix: string, ircClientConfig: IrcClientConfig) { + if (ircClientConfig.getIpv6Address()) { + log.info( + "Using existing IPv6 address %s for %s", + ircClientConfig.getIpv6Address(), + ircClientConfig.getUserId() + ); + return ircClientConfig.getIpv6Address(); + } + if (this.counter === -1) { + log.info("Retrieving counter..."); + this.counter = await this.dataStore.getIpv6Counter(); + } + + // the bot user will not have a user ID + const id = ircClientConfig.getUserId() || ircClientConfig.getUsername(); + if (!id) { + return; + } + log.info("Enqueueing IPv6 generation request for %s", id); + await this.queue.enqueue(id, { + prefix: prefix, + ircClientConfig: ircClientConfig + }); + } + + public async process (prefix: string, ircClientConfig: IrcClientConfig) { + this.counter += 1; + + // insert : every 4 characters from the end of the string going to the start + // e.g. 1a2b3c4d5e6 => 1a2:b3c4:d5e6 + const suffix = this.counter.toString(16).replace(/\B(?=(.{4})+(?!.))/g, ':'); + const address = prefix + suffix; + + let config = ircClientConfig; + config.setIpv6Address(address); + + const userId = ircClientConfig.getUserId(); + // we only want to persist the IPv6 address for real matrix users + if (userId) { + const existingConfig = await this.dataStore.getIrcClientConfig( + userId, ircClientConfig.getDomain() + ); + if (existingConfig) { + config = existingConfig; + config.setIpv6Address(address); + } + log.info("Generated new IPv6 address %s for %s", address, config.getUserId()); + // persist to db here before releasing the lock on this request. + await this.dataStore.storeIrcClientConfig(config); + } + + await this.dataStore.setIpv6Counter(this.counter); + return config.getIpv6Address(); + } +} From 33dcd0d4e5ebafdd02cc2611dfe86ed6c7517384 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 17:20:42 +0100 Subject: [PATCH 116/350] Linting --- src/irc/IdentGenerator.ts | 12 ++++++------ src/irc/Ipv6Generator.ts | 35 +++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/irc/IdentGenerator.ts b/src/irc/IdentGenerator.ts index 0a912ed0d..9f204b978 100644 --- a/src/irc/IdentGenerator.ts +++ b/src/irc/IdentGenerator.ts @@ -34,7 +34,7 @@ export class IdentGenerator { // We need to queue them because otherwise 2 clashing user_ids could be assigned // the same ident value (won't be in the database yet) this.queue = new Queue((item: unknown) => { - const {matrixUser, ircClientConfig} = item as { matrixUser: MatrixUser, ircClientConfig: IrcClientConfig}; + const {matrixUser, ircClientConfig} = item as { matrixUser: MatrixUser; ircClientConfig: IrcClientConfig}; return this.process(matrixUser, ircClientConfig); }); } @@ -50,7 +50,7 @@ export class IdentGenerator { */ public async getIrcNames(ircClientConfig: IrcClientConfig, matrixUser?: MatrixUser) { const username = ircClientConfig.getUsername(); - const info: {username?: string, realname: string} = { + const info: {username?: string; realname: string} = { username: undefined, realname: (matrixUser ? IdentGenerator.sanitiseRealname(matrixUser.getId()) : @@ -112,7 +112,7 @@ export class IdentGenerator { const existingConfig = await this.dataStore.getIrcClientConfig(matrixUser.getId(), configDomain); const config = existingConfig ? existingConfig : ircClientConfig; config.setUsername(uname); - + // persist to db here before releasing the lock on this request. await this.dataStore.storeIrcClientConfig(config); return config.getUsername(); @@ -159,7 +159,7 @@ export class IdentGenerator { * * return uname */ - var delim = "_"; + const delim = "_"; const modifyUsername = () => { if (uname.indexOf(delim) === -1) { uname = uname.substring(0, uname.length - 2) + delim + "1"; @@ -208,7 +208,7 @@ export class IdentGenerator { return uname; } - private static sanitiseUsername(username: string, replacementChar: string = "") { + private static sanitiseUsername(username: string, replacementChar = "") { username = username.toLowerCase(); // strip illegal chars according to RFC 1459 Sect 2.3.1 // (technically it's any ascii for but meh) @@ -223,7 +223,7 @@ export class IdentGenerator { } return username; } - + private static sanitiseRealname(realname: string) { // real name can be any old ASCII return realname.replace(/[^\x00-\x7F]/g, ""); diff --git a/src/irc/Ipv6Generator.ts b/src/irc/Ipv6Generator.ts index 25390dd7e..90ede6c15 100644 --- a/src/irc/Ipv6Generator.ts +++ b/src/irc/Ipv6Generator.ts @@ -1,5 +1,19 @@ +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -import Bluebird from "bluebird"; import { Queue } from "../util/Queue"; import { getLogger } from "../logging"; import { DataStore } from "../datastore/DataStore"; @@ -8,14 +22,14 @@ import { IrcClientConfig } from "../models/IrcClientConfig"; const log = getLogger("Ipv6Generator"); export class Ipv6Generator { - private counter: number = -1; + private counter = -1; private queue: Queue; constructor (private readonly dataStore: DataStore) { // Queue of ipv6 generation requests. // We need to queue them because otherwise 2 clashing user_ids could be assigned // the same ipv6 value (won't be in the database yet) this.queue = new Queue((item) => { - const processItem = item as {prefix: string, ircClientConfig: IrcClientConfig}; + const processItem = item as {prefix: string; ircClientConfig: IrcClientConfig}; return this.process(processItem.prefix, processItem.ircClientConfig); }); } @@ -32,14 +46,15 @@ export class Ipv6Generator { * @return {Promise} Resolves to the IPv6 address generated; the IPv6 address will * already be set on the given config. */ - public async generate (prefix: string, ircClientConfig: IrcClientConfig) { - if (ircClientConfig.getIpv6Address()) { + public async generate (prefix: string, ircClientConfig: IrcClientConfig): Promise { + const existingAddress = ircClientConfig.getIpv6Address(); + if (existingAddress) { log.info( "Using existing IPv6 address %s for %s", - ircClientConfig.getIpv6Address(), + existingAddress, ircClientConfig.getUserId() ); - return ircClientConfig.getIpv6Address(); + return existingAddress; } if (this.counter === -1) { log.info("Retrieving counter..."); @@ -49,13 +64,13 @@ export class Ipv6Generator { // the bot user will not have a user ID const id = ircClientConfig.getUserId() || ircClientConfig.getUsername(); if (!id) { - return; + throw Error("Neither a userId or username were provided to generate."); } log.info("Enqueueing IPv6 generation request for %s", id); - await this.queue.enqueue(id, { + return (await this.queue.enqueue(id, { prefix: prefix, ircClientConfig: ircClientConfig - }); + })) as string; } public async process (prefix: string, ircClientConfig: IrcClientConfig) { From 61ec7bfeaae2ad9fa07fe767f0d90cb8f3dd3c8f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 17:35:34 +0100 Subject: [PATCH 117/350] Convert formatting to Typescript --- package-lock.json | 41 +++++++++++ package.json | 8 ++- src/irc/{formatting.js => formatting.ts} | 92 ++++++++++++------------ src/models/IrcAction.ts | 2 +- src/models/MatrixAction.ts | 2 +- 5 files changed, 94 insertions(+), 51 deletions(-) rename src/irc/{formatting.js => formatting.ts} (84%) diff --git a/package-lock.json b/package-lock.json index b08135866..f69692f0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -147,12 +147,44 @@ "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.7.tgz", "integrity": "sha512-2Y8uPt0/jwjhQ6EiluT0XCri1Dbplr0ZxfFXUz+ye13gaqE8u5gL5ppao1JrUYr9cIip5S6MvQzBS7Kke7U9VA==" }, + "@types/domhandler": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@types/domhandler/-/domhandler-2.4.1.tgz", + "integrity": "sha512-cfBw6q6tT5sa1gSPFSRKzF/xxYrrmeiut7E0TxNBObiLSBTuFEHibcfEe3waQPEDbqBsq+ql/TOniw65EyDFMA==", + "dev": true + }, + "@types/domutils": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@types/domutils/-/domutils-1.7.2.tgz", + "integrity": "sha512-Nnwy1Ztwq42SSNSZSh9EXBJGrOZPR+PQ2sRT4VZy8hnsFXfCil7YlKO2hd2360HyrtFz2qwnKQ13ENrgXNxJbw==", + "dev": true, + "requires": { + "@types/domhandler": "*" + } + }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", "dev": true }, + "@types/he": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@types/he/-/he-1.1.0.tgz", + "integrity": "sha512-HyiLOiJhclRBPzcbYrNThdi0JOdq7bT4hq9jFBPQk4HGjzkwYVQnMj9IDi7qvYkg9QTly2oZ9kjm4j7d8Ic9eA==", + "dev": true + }, + "@types/htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/@types/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-fCxmHS4ryCUCfV9+CJZY1UjkbR+6Al/EQdX5Jh03qBj9gdlPG5q+7uNoDgE/ZNXb3XNWSAQgqKIWnbRCbOyyWA==", + "dev": true, + "requires": { + "@types/domhandler": "*", + "@types/domutils": "*", + "@types/node": "*" + } + }, "@types/json-schema": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.3.tgz", @@ -203,6 +235,15 @@ "moment": ">=2.14.0" } }, + "@types/sanitize-html": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.20.2.tgz", + "integrity": "sha512-SrefiiBebGIhxEFkpbbYOwO1S6+zQLWAC4s4tipchlHq1aO9bp0xiapM7Zm0ml20MF+3OePWYdksB1xtneKPxg==", + "dev": true, + "requires": { + "@types/htmlparser2": "*" + } + }, "@typescript-eslint/eslint-plugin": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.2.0.tgz", diff --git a/package.json b/package.json index 49a225c03..89478bc08 100644 --- a/package.json +++ b/package.json @@ -48,12 +48,14 @@ "winston-daily-rotate-file": "^3.2.1" }, "devDependencies": { - "@typescript-eslint/eslint-plugin": "^2.2.0", - "@typescript-eslint/parser": "^2.2.0", "@types/bluebird": "^3.5.27", + "@types/he": "^1.1.0", "@types/nedb": "^1.8.9", - "@types/pg": "^7.11.1", "@types/nopt": "^3.0.29", + "@types/pg": "^7.11.1", + "@types/sanitize-html": "^1.20.2", + "@typescript-eslint/eslint-plugin": "^2.2.0", + "@typescript-eslint/parser": "^2.2.0", "eslint": "^5.16.0", "jasmine": "^3.1.0", "nyc": "^14.1.1", diff --git a/src/irc/formatting.js b/src/irc/formatting.ts similarity index 84% rename from src/irc/formatting.js rename to src/irc/formatting.ts index 330f165eb..ea29e79fc 100644 --- a/src/irc/formatting.js +++ b/src/irc/formatting.ts @@ -1,8 +1,7 @@ -"use strict"; +import sanitizeHtml from "sanitize-html"; +import he from "he"; -const sanitizeHtml = require('sanitize-html'); -const he = require("he"); -const htmlNamesToColorCodes = { +const htmlNamesToColorCodes: {[color: string]: string[]} = { white: ['00', '0'], black: ['01', '1'], navy: ['02', '2'], @@ -22,7 +21,7 @@ const htmlNamesToColorCodes = { }; // These map the CSS color names to mIRC hex colors -const htmlNamesToHex = { +const htmlNamesToHex: {[color: string]: string} = { white: '#FFFFFF', black: '#000000', navy: '#00007F', @@ -42,10 +41,10 @@ const htmlNamesToHex = { }; // store the reverse mapping -var colorCodesToHtmlNames = {}; -var htmlNames = Object.keys(htmlNamesToColorCodes); -htmlNames.forEach(function(htmlName) { - htmlNamesToColorCodes[htmlName].forEach(function(colorNum) { +const colorCodesToHtmlNames: {[colorCode: string]: string} = {}; +const htmlNames = Object.keys(htmlNamesToColorCodes); +htmlNames.forEach((htmlName) => { + htmlNamesToColorCodes[htmlName].forEach((colorNum: string) => { colorCodesToHtmlNames[colorNum] = htmlNamesToHex[htmlName]; }); }); @@ -58,6 +57,12 @@ const STYLE_CODES = [STYLE_BOLD, STYLE_ITALICS, STYLE_UNDERLINE]; const RESET_CODE = '\u000f'; const REVERSE_CODE = '\u0016'; +interface StyleState { + color: string|null; + bcolor: string|null; + history: string[]; +} + /** * This is used as the default state for irc to html conversion. * The color attributes (color and bcolor) can be: @@ -66,13 +71,13 @@ const REVERSE_CODE = '\u0016'; * * @type {{color: (null|string), bcolor: (null|string), history: Array}} */ -const STYLE_DEFAULT_STATE = { +const STYLE_DEFAULT_STATE: StyleState = { "color": null, // The foreground colour. "bcolor": null, // The background colour. "history": [] // The history of opened tags. See the htmlTag function. }; -function escapeHtmlChars(text) { +export function escapeHtmlChars(text: string): string { return text .replace(/&/g, "&") .replace(//g, STYLE_BOLD], [//g, STYLE_UNDERLINE], [//g, STYLE_ITALICS], [//g, STYLE_BOLD], [//g, STYLE_ITALICS] ]; @@ -182,25 +184,25 @@ module.exports.htmlToIrc = function(html) { STYLE_COLOR + htmlNamesToColorCodes[htmlColor][0] ]); }); - for (var i = 0; i < replacements.length; i++) { - var rep = replacements[i]; + for (let i = 0; i < replacements.length; i++) { + const rep = replacements[i]; cleanHtml = cleanHtml.replace(rep[0], rep[1]); } // this needs a single pass through to fix up the reset codes, as they // 'close' all open tags. This pass through checks which tags are open and // then reopens them after a reset code. - var openStyleCodes = []; - var closeTagsToStyle = { + const openStyleCodes = []; + const closeTagsToStyle: {[tag: string]: string} = { "": STYLE_BOLD, "": STYLE_UNDERLINE, "": STYLE_ITALICS, "": STYLE_ITALICS, "": STYLE_BOLD }; - var closeTags = Object.keys(closeTagsToStyle); - var replacement; - for (i = 0; i < cleanHtml.length; i++) { - var ch = cleanHtml[i]; + const closeTags = Object.keys(closeTagsToStyle); + let replacement; + for (let i = 0; i < cleanHtml.length; i++) { + const ch = cleanHtml[i]; if (STYLE_CODES.indexOf(ch) >= 0) { openStyleCodes.push(ch); } @@ -213,8 +215,8 @@ module.exports.htmlToIrc = function(html) { i += (replacement.length - 1); } else { - for (var closeTagIndex = 0; closeTagIndex < closeTags.length; closeTagIndex++) { - var closeTag = closeTags[closeTagIndex]; + for (let closeTagIndex = 0; closeTagIndex < closeTags.length; closeTagIndex++) { + const closeTag = closeTags[closeTagIndex]; if (cleanHtml.indexOf(closeTag, i) === i) { // replace close tag with a reset and pop off the open // formatting code, then reopen remaining tags @@ -235,7 +237,7 @@ module.exports.htmlToIrc = function(html) { cleanHtml = cleanHtml.replace(/<[^>]+>/gm, ""); // unescape html characters - var escapeChars = [ + const escapeChars: [RegExp, string][] = [ [/>/g, '>'], [/</g, '<'], [/"/g, '"'], [/&/g, '&'] ]; escapeChars.forEach(function(escapeSet) { @@ -243,9 +245,9 @@ module.exports.htmlToIrc = function(html) { }); return cleanHtml; -}; +} -module.exports.ircToHtml = function(text) { +export function ircToHtml(text: string): string { // Escape HTML characters and add reset character to close all tags at the end. text = escapeHtmlChars(text) + RESET_CODE; @@ -280,7 +282,7 @@ module.exports.ircToHtml = function(text) { case REVERSE_CODE: // Swap the foreground and background colours. - let temp = state.color; + const temp = state.color; state.color = state.bcolor; state.bcolor = temp; // Close and re-open the font tag. @@ -321,24 +323,22 @@ module.exports.ircToHtml = function(text) { return tags; } }); -}; +} -module.exports.toIrcLowerCase = function(str, caseMapping) { - caseMapping = caseMapping || "rfc1459"; - var lower = str.toLowerCase(); +export function toIrcLowerCase(str: string, caseMapping: "strict-rfc1459"|"rfc1459" = "rfc1459") { + const lower = str.toLowerCase(); if (caseMapping === "rfc1459") { - lower = lower. + return lower. replace(/\[/g, "{"). replace(/\]/g, "}"). replace(/\\/g, "|"). replace(/\^/g, "~"); } else if (caseMapping === "strict-rfc1459") { - lower = lower. + return lower. replace(/\[/g, "{"). replace(/\]/g, "}"). replace(/\\/g, "|"); } - - return lower; -}; + throw Error("Unknown case mapping"); +} diff --git a/src/models/IrcAction.ts b/src/models/IrcAction.ts index b8faf37a4..bfce74f88 100644 --- a/src/models/IrcAction.ts +++ b/src/models/IrcAction.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import ircFormatting = require("../irc/formatting"); +import * as ircFormatting from "../irc/formatting"; const log = require("../logging").get("IrcAction"); import { MatrixAction } from "./MatrixAction"; diff --git a/src/models/MatrixAction.ts b/src/models/MatrixAction.ts index 2e0dc3223..04f4b9c80 100644 --- a/src/models/MatrixAction.ts +++ b/src/models/MatrixAction.ts @@ -111,7 +111,7 @@ export class MatrixAction { we need the plain text to match something.*/ let identifier; try { - identifier = (await intent.getProfileInfo(userId, 'displayname', true)).displayname; + identifier = (await intent.getProfileInfo(userId, 'displayname', true)).displayname || undefined; } catch (e) { // This shouldn't happen, but let's not fail to match if so. From 0a9b80de27efcb7a7a5b28883b33db39412f8ded Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 17:37:12 +0100 Subject: [PATCH 118/350] changelog --- changelog.d/832.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/832.misc diff --git a/changelog.d/832.misc b/changelog.d/832.misc new file mode 100644 index 000000000..996371c23 --- /dev/null +++ b/changelog.d/832.misc @@ -0,0 +1 @@ +Convert generator and formatter functions to Typescript \ No newline at end of file From 61f7cf80fc0f18f7c91e3d0bbc3274ff1f1f0caa Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 17:38:07 +0100 Subject: [PATCH 119/350] changelog --- changelog.d/831.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/831.misc diff --git a/changelog.d/831.misc b/changelog.d/831.misc new file mode 100644 index 000000000..d2895ced6 --- /dev/null +++ b/changelog.d/831.misc @@ -0,0 +1 @@ +Typescriptify BridgedClient and dependencies \ No newline at end of file From 349c166c020574f49c67cb8cb2bf4f5972468224 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 18:11:46 +0100 Subject: [PATCH 120/350] Split out ProcessedDict --- src/util/ProcessedDict.ts | 84 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/util/ProcessedDict.ts diff --git a/src/util/ProcessedDict.ts b/src/util/ProcessedDict.ts new file mode 100644 index 000000000..dd5a3e170 --- /dev/null +++ b/src/util/ProcessedDict.ts @@ -0,0 +1,84 @@ +import { LoggerInstance } from "winston"; + +/* +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const CLEANUP_TIME_MS = 1000 * 60 * 10; // 10min + +interface ProcessedSet { + [domain: string]: { + [hash: string]: { + nick: string, + ts: number|null, + } + } +}; + +export class ProcessedDict { + processed: ProcessedSet = {}; + private timeoutObj: NodeJS.Timeout|null = null; + constructor() { } + + public getClaimer(domain: string, hash: string) { + if (!this.processed[domain] || !this.processed[domain][hash]) { + return null; + } + return this.processed[domain][hash].nick; + } + + public claim(domain: string, hash: string, nick: string, cmd: string) { + if (!this.processed[domain]) { + this.processed[domain] = {}; + } + this.processed[domain][hash] = { + nick: nick, + // we don't ever want to purge NAMES events + ts: cmd === "names" ? null : Date.now() + }; + } + + public startCleaner (parentLog: LoggerInstance) { + const expiredList: {[domain: string]: string[] } = { }; + this.timeoutObj = setTimeout(() => { + const now = Date.now(); + // loop the processed list looking for entries older than CLEANUP_TIME_MS + Object.keys(this.processed).forEach((domain) => { + const entries = this.processed[domain]; + if (!entries) { return; } + Object.keys(entries).forEach((hash: string) => { + const entry = entries[hash]; + if (entry.ts && (entry.ts + CLEANUP_TIME_MS) < now) { + if (!expiredList[domain]) { + expiredList[domain] = []; + } + expiredList[domain].push(hash); + } + }); + }); + + // purge the entries + Object.keys(expiredList).forEach((domain) => { + const hashes = expiredList[domain]; + parentLog.debug("Cleaning up %s entries from %s", hashes.length, domain); + hashes.forEach((hash) => { + delete this.processed[domain][hash]; + }); + }); + + this.startCleaner(parentLog); + }, CLEANUP_TIME_MS); + } +} From c4b3e7f7a257652cb1700f1de14fdf0d8e4b822f Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Thu, 3 Oct 2019 18:12:19 +0100 Subject: [PATCH 121/350] Convert IrcEventBroker to Typescript --- src/irc/IrcEventBroker.js | 509 -------------------------------------- src/irc/IrcEventBroker.ts | 474 +++++++++++++++++++++++++++++++++++ 2 files changed, 474 insertions(+), 509 deletions(-) delete mode 100644 src/irc/IrcEventBroker.js create mode 100644 src/irc/IrcEventBroker.ts diff --git a/src/irc/IrcEventBroker.js b/src/irc/IrcEventBroker.js deleted file mode 100644 index 5d734d81c..000000000 --- a/src/irc/IrcEventBroker.js +++ /dev/null @@ -1,509 +0,0 @@ -/* - * This module contains all the logic to determine how incoming events from - * IRC clients are mapped to events which are passed to the bridge. - * - * For example, every connected IRC client will get messages down their TCP - * stream, but only 1 client should pass this through to the bridge to - * avoid duplicates. This is typically handled by the MatrixBridge which is a - * bot whose job it is to be the unique entity to have responsibility for passing - * these events through to the bridge. - * - * However, we support disabling the bridge entirely which means one of the many - * TCP streams needs to be responsible for passing the message to the bridge. - * This is done using the following algorithm: - * - Create a hash "H" of (prefix, command, command-parameters) (aka the line) - * - Does H exist in the "processed" list? - * * YES: Was it you who processed H before? - * * YES: Process it again (someone sent the same message twice). - * * NO: Ignore this message. (someone else has processed this) - * * NO: Add H to the "processed" list with your client associated with it - * (this works without racing because javascript is single-threaded) - * and pass the message to the bridge for processing. - * There are problems with this approach: - * - Unbounded memory consumption on the "processed" list. - * - Clients who previously "owned" messages disconnecting and not handling - * a duplicate messsage. - * These are fixed by: - * - Periodically culling the "processed" list after a time T. - * - Checking if the client who claimed a message still has an active TCP - * connection to the server. If they do not have an active connection, the - * message hash can be "stolen" by another client. - * - * Rationale - * --------- - * In an ideal world, we'd have unique IDs on each message and it'd be first come, - * first serve to claim an incoming message, but IRC doesn't "do" unique IDs. - * - * As a result, we need to handle the case where we get a message down that looks - * exactly like one that was previously handled. Handling this across clients is - * impossible (every message comes down like this, appearing as dupes). Handling - * this *within* a client is possible; the *SAME* client which handled the prev - * message knows that this isn't a dupe because dupes aren't sent down the same - * TCP connection. - * - * Handling messages like this is risky though. We don't know for sure if the - * client that handled the prev message will handle this new message. Therefore, - * we check if the client who did the prev message is "dead" (inactive TCP conn), - * and then "steal" ownership of that message if it is dead (again, this is - * thread-safe provided the check and steal is done on a single turn of the event - * loop). Even this isn't perfect though, as the connection may die without us - * being aware of it (before TCP/app timeouts kick in), so we want to avoid having - * to rely on stealing messages. - * - * We use a hashing algorithm mainly to reduce the key length per message - * (which would otherwise be max 510 bytes). The strength of the hash (randomness) - * determines the reliability of the bridge because it determines the rate of - * "stealing" that is performed. At the moment, a max key size of 510 bytes is - * acceptable with our expected rate of messages, so we're using the identity - * function as our hash algorithm. - * - * Determining when to remove these keys from the processed dict is Hard. We can't - * just mark it off when "all clients" get the message because all clients MAY NOT - * always get the message e.g. due to a disconnect (leading to dead keys which - * are never collected). Timeouts are reasonable but they need to be > TCP level - * MSL (worse case) assuming the IRCd in question doesn't store-and-forward. The - * MSL is typically 2 minutes, so a collection interval of 10 minutes is long - * enough. - */ - -"use strict"; -const { IrcAction } = require("../models/IrcAction"); -const { IrcUser } = require("../models/IrcUser"); -const { BridgeRequest } = require("../models/BridgeRequest"); -const log = require("../logging").get("IrcEventBroker"); - -const CLEANUP_TIME_MS = 1000 * 60 * 10; // 10min - -function ProcessedDict() { - this.processed = { - // server.domain: { - // hash: { - // nick: , - // ts: