diff --git a/changelog.d/1663.feature b/changelog.d/1663.feature new file mode 100644 index 000000000..7092275bf --- /dev/null +++ b/changelog.d/1663.feature @@ -0,0 +1,2 @@ +- New PM rooms are configured to disable calls, reactions, redactions, and stickers; + as they could not be bridged anyway. diff --git a/changelog.d/1709.bugfix b/changelog.d/1709.bugfix new file mode 100644 index 000000000..b22188042 --- /dev/null +++ b/changelog.d/1709.bugfix @@ -0,0 +1 @@ +Fix the bridge pooling so it supports TLS. \ No newline at end of file diff --git a/changelog.d/1711.bugfix b/changelog.d/1711.bugfix new file mode 100644 index 000000000..3dfd1ae74 --- /dev/null +++ b/changelog.d/1711.bugfix @@ -0,0 +1 @@ +Fix setup widget failing to authenticate. diff --git a/changelog.d/1715.bugfix b/changelog.d/1715.bugfix new file mode 100644 index 000000000..3cca2e732 --- /dev/null +++ b/changelog.d/1715.bugfix @@ -0,0 +1 @@ +Sort the list of channels in !listrooms output. diff --git a/changelog.d/1717.bugfix b/changelog.d/1717.bugfix new file mode 100644 index 000000000..f5939bd57 --- /dev/null +++ b/changelog.d/1717.bugfix @@ -0,0 +1 @@ +Fix cases where the IRC bridge may erronously believe a user is not joined to a channel in pooling mode. \ No newline at end of file diff --git a/changelog.d/1720.bugfix b/changelog.d/1720.bugfix new file mode 100644 index 000000000..7efdc6154 --- /dev/null +++ b/changelog.d/1720.bugfix @@ -0,0 +1 @@ +Ensure that all passwords can be decrypted on startup, to detect any issues with the provided passkey. diff --git a/package.json b/package.json index 27383b11b..d6a705641 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:app": "tsc --project ./tsconfig.json", "build:widget": "vite build --config widget/vite.config.ts", "dev:widget": "vite dev --config widget/vite.config.ts", - "test": "BLUEBIRD_DEBUG=1 ts-node --project spec/tsconfig.json node_modules/jasmine/bin/jasmine --stop-on-failure=true", + "test": "ts-node --project spec/tsconfig.json node_modules/jasmine/bin/jasmine --stop-on-failure=true", "test:e2e": "jest --config spec/e2e/jest.config.js --forceExit", "lint": "eslint -c .eslintrc --max-warnings 0 'spec/**/*.js' 'src/**/*.ts' && eslint -c ./widget/.eslintrc.js 'widget/src/**/*.{ts,tsx}'", "check": "yarn test && yarn lint", @@ -45,8 +45,8 @@ "logform": "^2.4.2", "matrix-appservice-bridge": "^9.0.0", "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.6-element.1", - "matrix-org-irc": "^2.0.0", - "matrix-widget-api": "^1.1.1", + "matrix-org-irc": "^2.0.1", + "matrix-widget-api": "^1.4.0", "nopt": "^6.0.0", "p-queue": "^6.6.2", "pg": "^8.8.0", diff --git a/spec/e2e/basic.spec.ts b/spec/e2e/basic.spec.ts index 2392bdd5e..6215786da 100644 --- a/spec/e2e/basic.spec.ts +++ b/spec/e2e/basic.spec.ts @@ -1,4 +1,6 @@ +import { TestIrcServer } from "matrix-org-irc"; import { IrcBridgeE2ETest } from "../util/e2e-test"; +import { describe, expect, it } from "@jest/globals"; describe('Basic bridge usage', () => { @@ -11,33 +13,36 @@ describe('Basic bridge usage', () => { await testEnv.setUp(); }); afterEach(() => { - return testEnv.tearDown(); + return testEnv?.tearDown(); }); it('should be able to dynamically bridge a room via the !join command', async () => { - const { homeserver, ircBridge } = testEnv; + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + const { homeserver } = testEnv; const alice = homeserver.users[0].client; const { bob } = testEnv.ircTest.clients; - await bob.join('#test'); - const adminRoomId = await alice.createRoom({ - is_direct: true, - invite: [ircBridge.appServiceUserId], - }); - await alice.waitForRoomEvent( - {eventType: 'm.room.member', sender: ircBridge.appServiceUserId, roomId: adminRoomId} - ); - await alice.sendText(adminRoomId, `!join #test`); - const invite = await alice.waitForRoomInvite( - {sender: ircBridge.appServiceUserId} - ); - const cRoomId = invite.roomId; - await alice.joinRoom(cRoomId); + // Create the channel + await bob.join(channel); + + const adminRoomId = await testEnv.createAdminRoomHelper(alice); + const cRoomId = await testEnv.joinChannelHelper(alice, adminRoomId, channel); const roomName = await alice.getRoomStateEvent(cRoomId, 'm.room.name', ''); - expect(roomName.name).toEqual('#test'); + expect(roomName.name).toEqual(channel); + // And finally wait for bob to appear. const bobUserId = `@irc_${bob.nick}:${homeserver.domain}`; await alice.waitForRoomEvent( {eventType: 'm.room.member', sender: bobUserId, stateKey: bobUserId, roomId: cRoomId} ); + + // Send some messages + const aliceMsg = bob.waitForEvent('message', 10000); + const bobMsg = alice.waitForRoomEvent( + {eventType: 'm.room.message', sender: bobUserId, roomId: cRoomId} + ); + alice.sendText(cRoomId, "Hello bob!"); + await aliceMsg; + bob.say(channel, "Hi alice!"); + await bobMsg; }); }); diff --git a/spec/e2e/membership.spec.ts b/spec/e2e/membership.spec.ts new file mode 100644 index 000000000..2d5dedc57 --- /dev/null +++ b/spec/e2e/membership.spec.ts @@ -0,0 +1,130 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { TestIrcServer } from "matrix-org-irc"; +import { IrcBridgeE2ETest } from "../util/e2e-test"; +import { describe, expect, it } from "@jest/globals"; + +const MEMBERSHIP_TIMEOUT = 3000; + +describe('Ensure membership is synced to IRC rooms', () => { + let testEnv: IrcBridgeE2ETest; + beforeEach(async () => { + testEnv = await IrcBridgeE2ETest.createTestEnv({ + matrixLocalparts: ['alice'], + ircNicks: ['bob', 'charlie', 'basil'].flatMap(nick => Array.from({length: 1}, (_, i) => `${nick}${i}`)), + }); + await testEnv.setUp(); + }); + afterEach(() => { + return testEnv?.tearDown(); + }); + it('ensure IRC puppets join', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const clients = Object.values(testEnv.ircTest.clients) + .map(client => ({userId: `@irc_${client.nick}:${homeserver.domain}`, client})); + const creatorClient = clients.pop()!; + + // Create the channel + await creatorClient.client.join(channel); + + const cRoomId = await testEnv.joinChannelHelper(alice, await testEnv.createAdminRoomHelper(alice), channel); + + const joinPromises: Promise[] = []; + + // Join all the users, and check all the membership events appear. + for (const ircUser of clients) { + joinPromises.push( + alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: ircUser.userId, stateKey: ircUser.userId, roomId: cRoomId}, + MEMBERSHIP_TIMEOUT, + ).then(({data}) => expect(data.content.membership).toEqual("join")) + ) + await ircUser.client.join(channel); + } + + await Promise.all(joinPromises); + }); + + it('ensure IRC puppets leave on part', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const clients = Object.values(testEnv.ircTest.clients) + .map(client => ({userId: `@irc_${client.nick}:${homeserver.domain}`, client})); + const creatorClient = clients.pop()!; + + // Create the channel + await creatorClient.client.join(channel); + + const cRoomId = await testEnv.joinChannelHelper(alice, await testEnv.createAdminRoomHelper(alice), channel); + + const joinPromises: Promise[] = []; + + // Join all the users, and check all the membership events appear. + for (const ircUser of clients) { + joinPromises.push( + alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: ircUser.userId, stateKey: ircUser.userId, roomId: cRoomId}, + MEMBERSHIP_TIMEOUT, + ).then(({data}) => expect(data.content.membership).toEqual("join")) + ) + await ircUser.client.join(channel); + } + + await Promise.all(joinPromises); + const partPromises: Promise[] = []; + + for (const ircUser of clients) { + partPromises.push( + alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: ircUser.userId, stateKey: ircUser.userId, roomId: cRoomId}, + MEMBERSHIP_TIMEOUT, + ).then(({data}) => expect(data.content.membership).toEqual("leave")) + ) + await ircUser.client.part(channel, 'getting out of here!'); + } + await Promise.all(partPromises); + }); + + it('ensure IRC puppets leave on quit', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const clients = Object.values(testEnv.ircTest.clients) + .map(client => ({userId: `@irc_${client.nick}:${homeserver.domain}`, client})); + const creatorClient = clients.pop()!; + + // Create the channel + await creatorClient.client.join(channel); + + const cRoomId = await testEnv.joinChannelHelper(alice, await testEnv.createAdminRoomHelper(alice), channel); + + const joinPromises: Promise[] = []; + + // Join all the users, and check all the membership events appear. + for (const ircUser of clients) { + joinPromises.push( + alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: ircUser.userId, stateKey: ircUser.userId, roomId: cRoomId}, + MEMBERSHIP_TIMEOUT, + ).then(({data}) => expect(data.content.membership).toEqual("join")) + ) + await ircUser.client.join(channel); + } + + await Promise.all(joinPromises); + const partPromises: Promise[] = []; + + for (const ircUser of clients) { + partPromises.push( + alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: ircUser.userId, stateKey: ircUser.userId, roomId: cRoomId}, + MEMBERSHIP_TIMEOUT, + ).then(({data}) => expect(data.content.membership).toEqual("leave")) + ) + await ircUser.client.disconnect(); + } + await Promise.all(partPromises); + }); +}); diff --git a/spec/e2e/pooling.spec.ts b/spec/e2e/pooling.spec.ts new file mode 100644 index 000000000..78e267f04 --- /dev/null +++ b/spec/e2e/pooling.spec.ts @@ -0,0 +1,119 @@ +import { TestIrcServer } from "matrix-org-irc"; +import { IrcBridgeE2ETest } from "../util/e2e-test"; +import { describe, it } from "@jest/globals"; + +const describeif = IrcBridgeE2ETest.usingRedis ? describe : describe.skip; + +describeif('Connection pooling', () => { + let testEnv: IrcBridgeE2ETest; + + beforeEach(async () => { + // Initial run of the bridge to setup a testing environment + testEnv = await IrcBridgeE2ETest.createTestEnv({ + matrixLocalparts: ['alice'], + ircNicks: ['bob'], + config: { + connectionPool: { + redisUrl: 'unused', + persistConnectionsOnShutdown: true, + } + } + }); + await testEnv.setUp(); + }) + + // Ensure we always tear down + afterEach(() => { + return testEnv.tearDown(); + }); + + it('should be able to shut down the bridge and start back up again', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const { bob } = testEnv.ircTest.clients; + + // Create the channel + await bob.join(channel); + + const adminRoomId = await testEnv.createAdminRoomHelper(alice); + const cRoomId = await testEnv.joinChannelHelper(alice, adminRoomId, channel); + + // And finally wait for bob to appear. + const bobUserId = `@irc_${bob.nick}:${homeserver.domain}`; + await alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: bobUserId, stateKey: bobUserId, roomId: cRoomId} + ); + + + // Send some messages + let aliceMsg = bob.waitForEvent('message', 10000); + let bobMsg = alice.waitForRoomEvent( + {eventType: 'm.room.message', sender: bobUserId, roomId: cRoomId} + ); + alice.sendText(cRoomId, "Hello bob!"); + await aliceMsg; + bob.say(channel, "Hi alice!"); + await bobMsg; + + console.log('Recreating bridge'); + + // Now kill the bridge, do NOT kill the dependencies. + await testEnv.recreateBridge(); + await testEnv.setUp(); + + aliceMsg = bob.waitForEvent('message', 10000); + bobMsg = alice.waitForRoomEvent( + {eventType: 'm.room.message', sender: bobUserId, roomId: cRoomId} + ); + alice.sendText(cRoomId, "Hello bob!"); + await aliceMsg; + bob.say(channel, "Hi alice!"); + await bobMsg; + }); + + it('should be able to recover from legacy client state', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const { bob } = testEnv.ircTest.clients; + + // Create the channel + await bob.join(channel); + + const adminRoomId = await testEnv.createAdminRoomHelper(alice); + const cRoomId = await testEnv.joinChannelHelper(alice, adminRoomId, channel); + + // And finally wait for bob to appear. + const bobUserId = `@irc_${bob.nick}:${homeserver.domain}`; + await alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: bobUserId, stateKey: bobUserId, roomId: cRoomId} + ); + + + // Send some messages + let aliceMsg = bob.waitForEvent('message', 10000); + let bobMsg = alice.waitForRoomEvent( + {eventType: 'm.room.message', sender: bobUserId, roomId: cRoomId} + ); + alice.sendText(cRoomId, "Hello bob!"); + await aliceMsg; + bob.say(channel, "Hi alice!"); + await bobMsg; + + // Now kill the bridge, do NOT kill the dependencies. + await testEnv.recreateBridge(); + await testEnv.setUp(); + + aliceMsg = bob.waitForEvent('message', 10000); + bobMsg = alice.waitForRoomEvent( + {eventType: 'm.room.message', sender: bobUserId, roomId: cRoomId} + ); + alice.sendText(cRoomId, "Hello bob!"); + await aliceMsg; + bob.say(channel, "Hi alice!"); + await bobMsg; + }); +}); diff --git a/spec/e2e/powerlevels.spec.ts b/spec/e2e/powerlevels.spec.ts new file mode 100644 index 000000000..d255970b6 --- /dev/null +++ b/spec/e2e/powerlevels.spec.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { TestIrcServer } from "matrix-org-irc"; +import { IrcBridgeE2ETest } from "../util/e2e-test"; +import { describe, expect, it } from "@jest/globals"; +import { PowerLevelContent } from "matrix-appservice-bridge"; + + +describe('Ensure powerlevels are appropriately applied', () => { + let testEnv: IrcBridgeE2ETest; + beforeEach(async () => { + testEnv = await IrcBridgeE2ETest.createTestEnv({ + matrixLocalparts: ['alice'], + ircNicks: ['bob', 'charlie'], + }); + await testEnv.setUp(); + }); + afterEach(() => { + return testEnv?.tearDown(); + }); + it('should update powerlevel of IRC user when OPed by an IRC user', async () => { + const channel = `#${TestIrcServer.generateUniqueNick("test")}`; + const { homeserver } = testEnv; + const alice = homeserver.users[0].client; + const { bob, charlie } = testEnv.ircTest.clients; + const charlieUserId = `@irc_${charlie.nick}:${homeserver.domain}`; + + // Create the channel + await bob.join(channel); + + const cRoomId = await testEnv.joinChannelHelper(alice, await testEnv.createAdminRoomHelper(alice), channel); + + // Now have charlie join and be opped. + await charlie.join(channel); + await alice.waitForRoomEvent( + {eventType: 'm.room.member', sender: charlieUserId, stateKey: charlieUserId, roomId: cRoomId} + ); + await bob.send('MODE', channel, '+o', charlie.nick); + const powerLevel = alice.waitForRoomEvent( + {eventType: 'm.room.power_levels', roomId: cRoomId, sender: testEnv.ircBridge.appServiceUserId} + ); + + const powerlevelContent = (await powerLevel).data.content; + console.log(powerlevelContent.users); + expect(powerlevelContent.users![charlieUserId]).toEqual( + testEnv.ircBridge.config.ircService.servers.localhost.modePowerMap!.o + ); + }); +}); diff --git a/spec/integ/pm.spec.js b/spec/integ/pm.spec.js index 48aff4f46..47f35ef0f 100644 --- a/spec/integ/pm.spec.js +++ b/spec/integ/pm.spec.js @@ -284,9 +284,19 @@ describe("IRC-to-Matrix PMing", () => { "m.room.canonical_alias": 100, "m.room.history_visibility": 100, "m.room.power_levels": 100, - "m.room.encryption": 100 + "m.room.encryption": 100, + "org.matrix.msc3401.call": 100, + "org.matrix.msc3401.call.member": 100, + "im.vector.modular.widgets": 100, + "io.element.voice_broadcast_info": 100, + "m.call.invite": 100, + "m.call.candidate": 100, + "m.reaction": 100, + "m.room.redaction": 100, + "m.sticker": 100, }, - invite: 100 + invite: 100, + redact: 100, }, }]); resolve(); diff --git a/spec/unit/pool-service/IrcClientRedisState.ts b/spec/unit/pool-service/IrcClientRedisState.ts new file mode 100644 index 000000000..e14eff3b4 --- /dev/null +++ b/spec/unit/pool-service/IrcClientRedisState.ts @@ -0,0 +1,107 @@ +import { DefaultIrcSupported } from "matrix-org-irc"; +import { IrcClientRedisState, IrcClientStateDehydrated } from "../../../src/pool-service/IrcClientRedisState"; + +const userId = "@foo:bar"; + +function fakeRedis(existingData: string|null = null): any { + return { + async hget(key, clientId) { + if (clientId !== userId) { + throw Error('Wrong user!'); + } + return existingData; + } + } +} + +const EXISTING_STATE: IrcClientStateDehydrated = { + loggedIn: true, + registered: true, + currentNick: "alice", + whoisData: [], + nickMod: 0, + modeForPrefix: { + 50: 'o', + }, + capabilities: { + serverCapabilites: ['some'], + serverCapabilitesSasl: ['caps'], + userCapabilites: ['for'], + userCapabilitesSasl: [] + }, + supportedState: DefaultIrcSupported, + hostMask: "", + chans: [ + ['fibble', { + key: '', + serverName: 'egg', + users: [ + ['bob', 'o'] + ], + mode: 'a', + modeParams: [ + ['o', ['bob']] + ] + }] + ], + prefixForMode: { + '+': 'o', + }, + maxLineLength: 100, + lastSendTime: 12345, +} + +describe("IrcClientRedisState", () => { + it("should be able to create a fresh state", async () => { + const state = await IrcClientRedisState.create( + fakeRedis(), + userId + ); + expect(state.loggedIn).toBeFalse(); + expect(state.registered).toBeFalse(); + expect(state.chans.size).toBe(0); + }); + it("should be able to load existing state", async () => { + const state = await IrcClientRedisState.create( + fakeRedis(JSON.stringify(EXISTING_STATE)), + userId + ); + expect(state.loggedIn).toBeTrue(); + expect(state.registered).toBeTrue(); + expect(state.chans.size).toBe(1); + console.log(state); + }); + it('should be able to repair previously buggy state', async () => { + const existingState = { + ...EXISTING_STATE, + chans: [ + [ + "#matrix-bridge-test", + { + "key": "#matrix-bridge-test", + "serverName": "#matrix-bridge-test", + "users": {}, + "mode": "+Cnst", + "modeParams": {}, + "created": "1683732619" + } + ], + [ + "#halfy-plumbs", + { + "key": "#halfy-plumbs", + "serverName": "#halfy-plumbs", + "users": {}, + "mode": "+Cnst", + "modeParams": {}, + "created": "1683732619" + } + ], + ] + } + const state = await IrcClientRedisState.create( + fakeRedis(JSON.stringify(existingState)), + userId + ); + }) +}); diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index de247bb62..8f918212f 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -27,8 +27,88 @@ interface Opts { config?: Partial, } +export class E2ETestMatrixClient extends MatrixClient { + + public async waitForRoomEvent>( + opts: {eventType: string, sender: string, roomId?: string, stateKey?: string}, + timeout = WAIT_EVENT_TIMEOUT, + ): Promise<{roomId: string, data: { + sender: string, type: string, state_key?: string, content: T, event_id: string, + }}> { + const {eventType, sender, roomId, stateKey} = opts; + return this.waitForEvent('room.event', (eventRoomId: string, eventData: { + sender: string, type: string, state_key?: string, content: T, event_id: string, + }) => { + if (eventData.sender !== sender) { + return undefined; + } + if (eventData.type !== eventType) { + return undefined; + } + if (roomId && eventRoomId !== roomId) { + return undefined; + } + if (stateKey !== undefined && eventData.state_key !== stateKey) { + return undefined; + } + const body = 'body' in eventData.content && eventData.content.body; + console.info( + // eslint-disable-next-line max-len + `${eventRoomId} ${eventData.event_id} ${eventData.type} ${eventData.sender} ${eventData.state_key ?? body ?? ''}` + ); + return {roomId: eventRoomId, data: eventData}; + }, `Timed out waiting for ${eventType} from ${sender} in ${roomId || "any room"}`, timeout) + } + + public async waitForRoomInvite( + opts: {sender: string, roomId?: string} + ): Promise<{roomId: string, data: unknown}> { + const {sender, roomId} = opts; + return this.waitForEvent('room.invite', (eventRoomId: string, eventData: { + sender: string + }) => { + const inviteSender = eventData.sender; + console.info(`Got invite to ${eventRoomId} from ${inviteSender}`); + if (eventData.sender !== sender) { + return undefined; + } + if (roomId && eventRoomId !== roomId) { + return undefined; + } + return {roomId: eventRoomId, data: eventData}; + }, `Timed out waiting for invite to ${roomId || "any room"} from ${sender}`) + } + + public async waitForEvent( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + emitterType: string, filterFn: (...args: any[]) => T|undefined, timeoutMsg: string, + timeout = WAIT_EVENT_TIMEOUT) + : Promise { + return new Promise((resolve, reject) => { + // eslint-disable-next-line prefer-const + let timer: NodeJS.Timeout; + const fn = (...args: unknown[]) => { + const data = filterFn(...args); + if (data) { + clearTimeout(timer); + resolve(data); + } + }; + timer = setTimeout(() => { + this.removeListener(emitterType, fn); + reject(new Error(timeoutMsg)); + }, timeout); + this.on(emitterType, fn) + }); + } +} + export class IrcBridgeE2ETest { + public static get usingRedis() { + return !!IRCBRIDGE_TEST_REDIS_URL; + } + private static async createDatabase() { const pgClient = new PgClient(`${process.env.IRCBRIDGE_TEST_PGURL}/postgres`); try { @@ -41,25 +121,48 @@ export class IrcBridgeE2ETest { await pgClient.end(); } } + static async createTestEnv(opts: Opts = {}): Promise { + const workerID = parseInt(process.env.JEST_WORKER_ID ?? '0'); const { matrixLocalparts, config } = opts; const ircTest = new TestIrcServer(); const [postgresDb, homeserver] = await Promise.all([ this.createDatabase(), - createHS(["ircbridge_bot", ...matrixLocalparts || []]), + createHS(["ircbridge_bot", ...matrixLocalparts || []], workerID), ircTest.setUp(opts.ircNicks), ]); + const redisUri = IRCBRIDGE_TEST_REDIS_URL && `${IRCBRIDGE_TEST_REDIS_URL}/${workerID}`; let redisPool: IrcConnectionPool|undefined; - if (IRCBRIDGE_TEST_REDIS_URL) { + if (redisUri) { redisPool = new IrcConnectionPool({ - redisUri: IRCBRIDGE_TEST_REDIS_URL, + redisUri, metricsHost: false, metricsPort: 7002, loggingLevel: 'debug', }); } + const registration = AppServiceRegistration.fromObject({ + id: homeserver.id, + as_token: homeserver.appserviceConfig.asToken, + hs_token: homeserver.appserviceConfig.hsToken, + sender_localpart: homeserver.appserviceConfig.senderLocalpart, + namespaces: { + users: [{ + exclusive: true, + regex: `@irc_.+:${homeserver.domain}`, + }], + // TODO: No support on complement yet: + // https://github.com/matrix-org/complement/blob/8e341d54bbb4dbbabcea25e6a13b29ead82978e3/internal/docker/builder.go#L413 + aliases: [{ + exclusive: true, + regex: `#irc_.+:${homeserver.domain}`, + }] + }, + url: "not-used", + }); + const ircBridge = new IrcBridge({ homeserver: { domain: homeserver.domain, @@ -72,6 +175,9 @@ export class IrcBridgeE2ETest { connectionString: `${process.env.IRCBRIDGE_TEST_PGURL}/${postgresDb}`, }, ircService: { + ircHandler: { + powerLevelGracePeriodMs: 0, + }, servers: { localhost: { ...IrcServer.DEFAULT_CONFIG, @@ -129,44 +235,32 @@ export class IrcBridgeE2ETest { port: 0, } }, - ...(IRCBRIDGE_TEST_REDIS_URL && { connectionPool: { - redisUrl: IRCBRIDGE_TEST_REDIS_URL, + ...config, + ...(redisUri && { connectionPool: { persistConnectionsOnShutdown: false, + ...config?.connectionPool || {}, + redisUrl: redisUri, } }), - ...config, - }, AppServiceRegistration.fromObject({ - id: homeserver.id, - as_token: homeserver.appserviceConfig.asToken, - hs_token: homeserver.appserviceConfig.hsToken, - sender_localpart: homeserver.appserviceConfig.senderLocalpart, - namespaces: { - users: [{ - exclusive: true, - regex: `@irc_.+:${homeserver.domain}`, - }], - // TODO: No support on complement yet: - // https://github.com/matrix-org/complement/blob/8e341d54bbb4dbbabcea25e6a13b29ead82978e3/internal/docker/builder.go#L413 - aliases: [{ - exclusive: true, - regex: `#irc_.+:${homeserver.domain}`, - }] - }, - url: "not-used", - }), { - isDBInMemory: false, - }); - return new IrcBridgeE2ETest(homeserver, ircBridge, postgresDb, ircTest, redisPool) + }, registration); + return new IrcBridgeE2ETest(homeserver, ircBridge, registration, postgresDb, ircTest, redisPool) } private constructor( public readonly homeserver: ComplementHomeServer, - public readonly ircBridge: IrcBridge, - public readonly postgresDb: string, + public ircBridge: IrcBridge, + public readonly registration: AppServiceRegistration, + readonly postgresDb: string, public readonly ircTest: TestIrcServer, public readonly pool?: IrcConnectionPool) { } + public async recreateBridge() { + await this.ircBridge.kill('Recreating'); + this.ircBridge = new IrcBridge(this.ircBridge.config, this.registration); + return this.ircBridge; + } + private async dropDatabase() { if (!this.postgresDb) { // Database was never set up. @@ -185,82 +279,49 @@ export class IrcBridgeE2ETest { await this.ircBridge.run(null); } + private static async warnOnSlowTearDown(name: string, handler: () => Promise) { + const timeout = setTimeout(() => { + console.warn(`Teardown fn ${name} has taken over 5 seconds to complete`); + }, 5000); + try { + await handler(); + } + finally { + clearTimeout(timeout); + } + } + public async tearDown(): Promise { await Promise.allSettled([ - this.ircBridge?.kill(), - this.ircTest.tearDown(), - this.homeserver?.users.map(c => c.client.stop()), - this.homeserver && destroyHS(this.homeserver.id), - this.dropDatabase(), - this.pool?.close(), + IrcBridgeE2ETest.warnOnSlowTearDown('ircBridge.kill', () => this.ircBridge?.kill()), + // TODO: Skip teardown if the clients are already disconnected. + // IrcBridgeE2ETest.warnOnSlowTearDown('ircTest.tearDown', () => this.ircTest.tearDown()), + IrcBridgeE2ETest.warnOnSlowTearDown('homeserver.stop', + () => Promise.all(this.homeserver?.users.map(c => c.client.stop()))), + IrcBridgeE2ETest.warnOnSlowTearDown('destroyHS', () => destroyHS(this.homeserver.id)), + IrcBridgeE2ETest.warnOnSlowTearDown('dropDatabase', () => this.dropDatabase()), ]); - } -} - -export class E2ETestMatrixClient extends MatrixClient { - - public async waitForRoomEvent( - opts: {eventType: string, sender: string, roomId?: string, stateKey?: string} - ): Promise<{roomId: string, data: unknown}> { - const {eventType, sender, roomId, stateKey} = opts; - return this.waitForEvent('room.event', (eventRoomId: string, eventData: { - sender: string, type: string, state_key?: string, content: unknown - }) => { - console.info(`Got ${eventRoomId}`, eventData); - if (eventData.sender !== sender) { - return undefined; - } - if (eventData.type !== eventType) { - return undefined; - } - if (roomId && eventRoomId !== roomId) { - return undefined; - } - if (stateKey !== undefined && eventData.state_key !== stateKey) { - return undefined; - } - return {roomId: eventRoomId, data: eventData}; - }, `Timed out waiting for ${eventType} from ${sender} in ${roomId || "any room"}`) + await IrcBridgeE2ETest.warnOnSlowTearDown('pool.close()', async () => this.pool?.close()); } - public async waitForRoomInvite( - opts: {sender: string, roomId?: string} - ): Promise<{roomId: string, data: unknown}> { - const {sender, roomId} = opts; - return this.waitForEvent('room.invite', (eventRoomId: string, eventData: { - sender: string - }) => { - const inviteSender = eventData.sender; - console.info(`Got invite to ${eventRoomId} from ${inviteSender}`); - if (eventData.sender !== sender) { - return undefined; - } - if (roomId && eventRoomId !== roomId) { - return undefined; - } - return {roomId: eventRoomId, data: eventData}; - }, `Timed out waiting for invite to ${roomId || "any room"} from ${sender}`) + public async createAdminRoomHelper(client: E2ETestMatrixClient): Promise { + const adminRoomId = await client.createRoom({ + is_direct: true, + invite: [this.ircBridge.appServiceUserId], + }); + await client.waitForRoomEvent( + {eventType: 'm.room.member', sender: this.ircBridge.appServiceUserId, roomId: adminRoomId} + ); + return adminRoomId; } - public async waitForEvent( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - emitterType: string, filterFn: (...args: any[]) => T|undefined, timeoutMsg: string) - : Promise { - return new Promise((resolve, reject) => { - // eslint-disable-next-line prefer-const - let timer: NodeJS.Timeout; - const fn = (...args: unknown[]) => { - const data = filterFn(...args); - if (data) { - clearTimeout(timer); - resolve(data); - } - }; - timer = setTimeout(() => { - this.removeListener(emitterType, fn); - reject(new Error(timeoutMsg)); - }, WAIT_EVENT_TIMEOUT); - this.on(emitterType, fn) - }); + public async joinChannelHelper(client: E2ETestMatrixClient, adminRoomId: string, channel: string): Promise { + await client.sendText(adminRoomId, `!join ${channel}`); + const invite = await client.waitForRoomInvite( + {sender: this.ircBridge.appServiceUserId} + ); + const cRoomId = invite.roomId; + await client.joinRoom(cRoomId); + return cRoomId; } } diff --git a/spec/util/homerunner.ts b/spec/util/homerunner.ts index 06a79ceaf..5837a8e6c 100644 --- a/spec/util/homerunner.ts +++ b/spec/util/homerunner.ts @@ -25,9 +25,12 @@ export interface ComplementHomeServer { users: {userId: string, accessToken: string, deviceId: string, client: E2ETestMatrixClient}[] } -let appPort = 32100; +// Ensure we don't clash with other tests. async function waitForHomerunner() { + + // Check if port is in use. + // Needs https://github.com/matrix-org/complement/issues/398 let attempts = 0; do { @@ -47,10 +50,10 @@ async function waitForHomerunner() { } } -export async function createHS(localparts: string[] = []): Promise { +export async function createHS(localparts: string[] = [], workerId: number): Promise { + const appPort = 49152 + workerId; await waitForHomerunner(); // Ensure we never use the same port twice. - appPort++; const AppserviceConfig = { id: 'ircbridge', port: appPort, diff --git a/src/bridge/AdminRoomHandler.ts b/src/bridge/AdminRoomHandler.ts index 1fc9a5c28..2a13898c7 100644 --- a/src/bridge/AdminRoomHandler.ts +++ b/src/bridge/AdminRoomHandler.ts @@ -601,7 +601,7 @@ export class AdminRoomHandler { let chanList = `You are joined to ${client.chanList.size} rooms: \n\n`; let chanListHTML = `

You are joined to ${client.chanList.size} rooms:

    `; - for (const channel of client.chanList) { + for (const channel of [...client.chanList].sort()) { const rooms = await this.ircBridge.getStore().getMatrixRoomsForChannel(server, channel); chanList += `- \`${channel}\` which is bridged to ${rooms.map((r) => r.getId()).join(", ")}`; const roomMentions = rooms diff --git a/src/bridge/IrcBridge.ts b/src/bridge/IrcBridge.ts index 4032af4c0..5a620811f 100644 --- a/src/bridge/IrcBridge.ts +++ b/src/bridge/IrcBridge.ts @@ -597,6 +597,7 @@ export class IrcBridge { this.config.connectionPool.redisUrl, ); this.ircPoolClient.on('lostConnection', () => { + console.log('Lost connection to bridge'); this.kill(); }); await this.ircPoolClient.listen(); @@ -649,6 +650,8 @@ export class IrcBridge { throw Error("Incorrect database config"); } + await this.dataStore.ensurePasskeyCanDecrypt(); + await this.dataStore.removeConfigMappings(); if (this.activityTracker) { diff --git a/src/bridge/IrcHandler.ts b/src/bridge/IrcHandler.ts index d515b853e..7df4e5150 100644 --- a/src/bridge/IrcHandler.ts +++ b/src/bridge/IrcHandler.ts @@ -196,9 +196,22 @@ export class IrcHandler { "m.room.canonical_alias": 100, "m.room.history_visibility": 100, "m.room.power_levels": 100, - "m.room.encryption": 100 + "m.room.encryption": 100, + // Event types that we cannot translate to IRC; + // we might as well block them with PLs so + // Matrix clients can hide them from their UI. + "m.call.invite": 100, + "m.call.candidate": 100, + "org.matrix.msc3401.call": 100, + "org.matrix.msc3401.call.member": 100, + "im.vector.modular.widgets": 100, + "io.element.voice_broadcast_info": 100, + "m.reaction": 100, + "m.room.redaction": 100, + "m.sticker": 100, }, invite: 100, + redact: 100, }, type: "m.room.power_levels", state_key: "", diff --git a/src/datastore/DataStore.ts b/src/datastore/DataStore.ts index 0889fc98c..b1851ebc9 100644 --- a/src/datastore/DataStore.ts +++ b/src/datastore/DataStore.ts @@ -159,6 +159,8 @@ export interface DataStore extends ProvisioningStore { storeIrcClientConfig(config: IrcClientConfig): Promise; + ensurePasskeyCanDecrypt(): Promise; + getMatrixUserByLocalpart(localpart: string): Promise; getUserFeatures(userId: string): Promise; diff --git a/src/datastore/NedbDataStore.ts b/src/datastore/NedbDataStore.ts index 1b9bda1ba..38406d933 100644 --- a/src/datastore/NedbDataStore.ts +++ b/src/datastore/NedbDataStore.ts @@ -569,12 +569,15 @@ export class NeDBDataStore implements DataStore { } const clientConfig = new IrcClientConfig(userId, domain, configData); const encryptedPass = clientConfig.getPassword(); - if (encryptedPass) { - if (!this.cryptoStore) { - throw new Error(`Cannot decrypt password of ${userId} - no private key`); + if (encryptedPass && this.cryptoStore) { + // NOT fatal, but really worrying. + try { + const decryptedPass = this.cryptoStore.decrypt(encryptedPass); + clientConfig.setPassword(decryptedPass); + } + catch (ex) { + log.warn(`Failed to decrypt password for ${userId} ${domain}`, ex); } - const decryptedPass = this.cryptoStore.decrypt(encryptedPass); - clientConfig.setPassword(decryptedPass); } return clientConfig; } @@ -610,6 +613,29 @@ export class NeDBDataStore implements DataStore { await this.userStore.setMatrixUser(user); } + public async ensurePasskeyCanDecrypt(): Promise { + if (!this.cryptoStore) { + return; + } + const docs = await this.userStore.select({ + type: "matrix", + "data.client_config": {$exists: true}, + }); + for (const { id: userId, data } of docs) { + for (const [domain, clientConfig] of Object.entries(data.client_config)) { + if (clientConfig.password) { + try { + this.cryptoStore.decrypt(clientConfig.password); + } + catch (ex) { + log.error(`Failed to decrypt password for ${userId} on ${domain}`, ex); + throw Error('Cannot decrypt user password, refusing to continue', { cause: ex }); + } + } + } + } + } + public async getUserFeatures(userId: string): Promise { const matrixUser = await this.userStore.getMatrixUser(userId); return matrixUser ? (matrixUser.get("features") as UserFeatures || {}) : {}; diff --git a/src/datastore/postgres/PgDataStore.ts b/src/datastore/postgres/PgDataStore.ts index fe193efbe..4dbe3e8ec 100644 --- a/src/datastore/postgres/PgDataStore.ts +++ b/src/datastore/postgres/PgDataStore.ts @@ -516,7 +516,13 @@ export class PgDataStore implements DataStore, ProvisioningStore { const row = res.rows[0]; const config = row.config || {}; // This may not be defined. if (row.password && this.cryptoStore) { - config.password = this.cryptoStore.decrypt(row.password); + // NOT fatal, but really worrying. + try { + config.password = this.cryptoStore.decrypt(row.password); + } + catch (ex) { + log.warn(`Failed to decrypt password for ${userId} ${domain}`, ex); + } } return new IrcClientConfig(userId, domain, config); } @@ -545,6 +551,24 @@ export class PgDataStore implements DataStore, ProvisioningStore { await this.pgPool.query(statement, Object.values(parameters)); } + + public async ensurePasskeyCanDecrypt(): Promise { + if (!this.cryptoStore) { + return; + } + const res = await this.pgPool.query<{password: string, user_id: string, domain: string}>( + "SELECT password, user_id, domain FROM client_config WHERE password IS NOT NULL"); + for (const { password, user_id, domain } of res.rows) { + try { + this.cryptoStore.decrypt(password); + } + catch (ex) { + log.error(`Failed to decrypt password for ${user_id} on ${domain}`, ex); + throw Error('Cannot decrypt user password, refusing to continue', { cause: ex }); + } + } + } + public async getMatrixUserByLocalpart(localpart: string): Promise { const res = await this.pgPool.query("SELECT user_id, data FROM matrix_users WHERE user_id = $1", [ `@${localpart}:${this.bridgeDomain}`, diff --git a/src/irc/BridgedClient.ts b/src/irc/BridgedClient.ts index afbad2e88..80e269e61 100644 --- a/src/irc/BridgedClient.ts +++ b/src/irc/BridgedClient.ts @@ -808,6 +808,7 @@ export class BridgedClient extends EventEmitter { identResolver: () => void) { // If this state has carried over from a previous connection, pull in any channels. [...connInst.client.chans.keys()].forEach(k => this.chanList.add(k)); + console.log('Adding existing channels', this.chanList.entries()); // listen for a connect event which is done when the TCP connection is // established and set ident info (this is different to the connect() callback // in node-irc which actually fires on a registered event..) @@ -846,21 +847,40 @@ export class BridgedClient extends EventEmitter { `The error was: ${errType} ${errorMsg}` ); }); + + const discoverChannel = (channel: string) => { + // If this has happened, our state is horribly invalid. + if (channel.startsWith('#') && !connInst.client.chans.has(channel)) { + this.log.info(`Channel ${channel} not found in client state, but we got a message from the channel`); + connInst.client.chanData(channel, true); + this.chanList.add(channel); + } + } + connInst.client.on("join", (channel, nick) => { if (this.nick !== nick) { return; } log.debug(`Joined ${channel}`); this.chanList.add(channel); }); connInst.client.on("part", (channel, nick) => { - if (this.nick !== nick) { return; } + if (this.nick !== nick) { + discoverChannel(channel); + return; + } log.debug(`Parted ${channel}`); this.chanList.delete(channel); }); connInst.client.on("kick", (channel, nick) => { - if (this.nick !== nick) { return; } + if (this.nick !== nick) { + discoverChannel(channel); + return; + } log.debug(`Kicked from ${channel}`); this.chanList.delete(channel); }); + connInst.client.on("message", (from, channel) => { + discoverChannel(channel); + }) connInst.onDisconnect = (reason) => { this._disconnectReason = reason; diff --git a/src/irc/ConnectionInstance.ts b/src/irc/ConnectionInstance.ts index 54ed4e8a8..29e5068cf 100644 --- a/src/irc/ConnectionInstance.ts +++ b/src/irc/ConnectionInstance.ts @@ -352,7 +352,6 @@ export class ConnectionInstance { }); // decorate client.send to refresh the timer const realSend = this.client.send; - // eslint-disable-next-line @typescript-eslint/no-explicit-any this.client.send = (...args: string[]) => { keepAlivePing(); this.resetPingSendTimer(); // sending a message counts as a ping @@ -428,7 +427,7 @@ export class ConnectionInstance { // Returns: A promise which resolves to a ConnectionInstance const retryConnection = async () => { - + const domain = server.randomDomain(); const redisConn = opts.useRedisPool && await opts.useRedisPool.createOrGetIrcSocket(ident, { ...connectionOpts, clientId: ident, @@ -436,10 +435,11 @@ export class ConnectionInstance { localAddress: connectionOpts.localAddress ?? undefined, localPort: connectionOpts.localPort ?? undefined, family: connectionOpts.family ?? undefined, + host: domain, }); const nodeClient = new Client( - server.randomDomain(), opts.nick, connectionOpts, redisConn?.state, redisConn, + domain, opts.nick, connectionOpts, redisConn?.state, redisConn, ); const inst = new ConnectionInstance( nodeClient, server.domain, opts.nick, { diff --git a/src/pool-service/IrcClientRedisState.ts b/src/pool-service/IrcClientRedisState.ts index 01498e1ca..5804aad0c 100644 --- a/src/pool-service/IrcClientRedisState.ts +++ b/src/pool-service/IrcClientRedisState.ts @@ -1,12 +1,27 @@ import { Redis } from 'ioredis'; -import { ChanData, IrcClientState, WhoisResponse, - IrcCapabilities, IrcSupported, DefaultIrcSupported } from 'matrix-org-irc'; +import { IrcClientState, WhoisResponse, + IrcCapabilities, IrcSupported, ChanData } from 'matrix-org-irc'; import { REDIS_IRC_CLIENT_STATE_KEY } from './types'; import * as Logger from "../logging"; const log = Logger.get('IrcClientRedisState'); -interface IrcClientStateDehydrated { + +interface ChanDataDehydrated { + created?: string; + key: string; + serverName: string; + /** + * nick => mode + */ + users: [string, string][]; + mode: string; + modeParams: [string, string[]][]; + topic?: string; + topicBy?: string; +} + +export interface IrcClientStateDehydrated { loggedIn: boolean; registered: boolean; /** @@ -19,9 +34,9 @@ interface IrcClientStateDehydrated { [prefix: string]: string; }; capabilities: ReturnType; - supportedState: IrcSupported; + supportedState?: IrcSupported; hostMask: string; - chans: [string, ChanData][]; + chans?: [string, ChanDataDehydrated][]; prefixForMode: { [mode: string]: string; }; @@ -29,12 +44,36 @@ interface IrcClientStateDehydrated { lastSendTime: number; } + export class IrcClientRedisState implements IrcClientState { private putStatePromise: Promise = Promise.resolve(); - static async create(redis: Redis, clientId: string) { - const data = await redis.hget(REDIS_IRC_CLIENT_STATE_KEY, clientId); + static async create(redis: Redis, clientId: string, freshState: boolean) { + log.debug(`Requesting ${freshState ? "fresh" : "existing"} state for ${clientId}`); + const data = freshState ? null : await redis.hget(REDIS_IRC_CLIENT_STATE_KEY, clientId); const deseralisedData = data ? JSON.parse(data) as IrcClientStateDehydrated : {} as Record; + const chans = new Map(); + + // In a previous iteration we failed to seralise this properly. + deseralisedData.chans?.forEach(([channelName, chanData]) => { + const isBuggyState = !Array.isArray(chanData.users); + chans.set(channelName, { + ...chanData, + users: new Map(!isBuggyState ? chanData.users : []), + modeParams: new Map(!isBuggyState ? chanData.modeParams : []), + }) + }); + + // We also had a bug where the supported state is bloated enormously + if (deseralisedData.supportedState) { + deseralisedData.supportedState.channel.modes = { + a: [...new Set(deseralisedData.supportedState.channel.modes.a.split(''))].join(''), + b: [...new Set(deseralisedData.supportedState.channel.modes.b.split(''))].join(''), + c: [...new Set(deseralisedData.supportedState.channel.modes.c.split(''))].join(''), + d: [...new Set(deseralisedData.supportedState.channel.modes.d.split(''))].join(''), + } + deseralisedData.supportedState.extra = [...new Set(deseralisedData.supportedState.extra)]; + } // The client library is currently responsible for flushing any new changes // to the state so we do not need to detect changes in this class. @@ -47,16 +86,35 @@ export class IrcClientRedisState implements IrcClientState { whoisData: new Map(deseralisedData.whoisData), modeForPrefix: deseralisedData.modeForPrefix ?? { }, hostMask: deseralisedData.hostMask ?? '', - chans: new Map(deseralisedData.chans), + chans, maxLineLength: deseralisedData.maxLineLength ?? -1, lastSendTime: deseralisedData.lastSendTime ?? 0, prefixForMode: deseralisedData.prefixForMode ?? {}, - supportedState: deseralisedData.supportedState ?? DefaultIrcSupported, + supportedState: deseralisedData.supportedState ?? { + channel: { + idlength: {}, + length: 200, + limit: {}, + modes: { a: '', b: '', c: '', d: ''}, + types: '', + }, + kicklength: 0, + maxlist: {}, + maxtargets: {}, + modes: 3, + nicklength: 9, + topiclength: 0, + usermodes: '', + usermodepriority: '', // E.g "ov" + casemapping: 'ascii', + extra: [], + }, capabilities: new IrcCapabilities(deseralisedData.capabilities), }; return new IrcClientRedisState(redis, clientId, innerState); } + private constructor( private readonly redis: Redis, private readonly clientId: string, @@ -175,22 +233,34 @@ export class IrcClientRedisState implements IrcClientState { public flush() { + const chans: [string, ChanDataDehydrated][] = []; + this.innerState.chans.forEach((chanData, channelName) => { + chans.push([ + channelName, + { + ...chanData, + users: [...chanData.users.entries()], + modeParams: [...chanData.modeParams.entries()], + } + ]) + }); + const serialState = JSON.stringify({ ...this.innerState, whoisData: [...this.innerState.whoisData.entries()], - chans: [...this.innerState.chans.entries()], + chans, capabilities: this.innerState.capabilities.serialise(), supportedState: this.supportedState, } as IrcClientStateDehydrated); - this.putStatePromise = this.putStatePromise.catch((ex) => { - log.warn(`Failed to store state for ${this.clientId}`, ex); - }).finally(() => { - return this.innerPutState(serialState); + this.putStatePromise = this.putStatePromise.then(() => { + return this.innerPutState(serialState).catch((ex) => { + log.warn(`Failed to store state for ${this.clientId}`, ex); + }); }); } private async innerPutState(data: string) { - return this.redis.hset(REDIS_IRC_CLIENT_STATE_KEY, this.clientId, data); + await this.redis.hset(REDIS_IRC_CLIENT_STATE_KEY, this.clientId, data); } } diff --git a/src/pool-service/IrcConnectionPool.ts b/src/pool-service/IrcConnectionPool.ts index f341dfc47..19aa766ac 100644 --- a/src/pool-service/IrcConnectionPool.ts +++ b/src/pool-service/IrcConnectionPool.ts @@ -21,6 +21,8 @@ import { parseMessage } from 'matrix-org-irc'; import { collectDefaultMetrics, register, Gauge } from 'prom-client'; import { createServer, Server } from 'http'; +collectDefaultMetrics(); + const log = new Logger('IrcConnectionPool'); const TIME_TO_WAIT_BEFORE_PONG = 10000; const STREAM_HISTORY_MAXLEN = 50; @@ -52,8 +54,12 @@ export class IrcConnectionPool { private heartbeatTimer?: NodeJS.Timer; constructor(private readonly config: typeof Config) { + this.shouldRun = false; this.cmdWriter = new Redis(config.redisUri, { lazyConnect: true }); this.cmdReader = new Redis(config.redisUri, { lazyConnect: true }); + this.cmdWriter.on('connecting', () => { + log.debug('Connecting to', config.redisUri); + }); } private updateLastRead(lastRead: string) { @@ -74,7 +80,6 @@ export class IrcConnectionPool { } private async createConnectionForOpts(opts: ConnectionCreateArgs): Promise { - let socket: Socket; if (opts.secure) { let secureOpts: tls.ConnectionOptions = { ...opts, @@ -89,11 +94,12 @@ export class IrcConnectionPool { }; } - socket = await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { // Taken from https://github.com/matrix-org/node-irc/blob/0764733af7c324ee24f8c2a3c26fe9d1614be344/src/irc.ts#L1231 const sock = tls.connect(secureOpts, () => { if (sock.authorized) { resolve(sock); + return; } let valid = false; const err = sock.authorizationError.toString(); @@ -125,7 +131,7 @@ export class IrcConnectionPool { }); } return new Promise((resolve, reject) => { - socket = createConnection(opts, () => resolve(socket)) as Socket; + const socket = createConnection(opts, () => resolve(socket)) as Socket; socket.once('error', (error) => { reject(error); }); @@ -329,8 +335,12 @@ export class IrcConnectionPool { } public async start() { + if (this.shouldRun) { + // Is already running! + return; + } + this.shouldRun = true; Logger.configure({ console: this.config.loggingLevel }); - collectDefaultMetrics(); // Load metrics if (this.config.metricsHost) { diff --git a/src/pool-service/IrcPoolClient.ts b/src/pool-service/IrcPoolClient.ts index dde1ba308..54d751f6f 100644 --- a/src/pool-service/IrcPoolClient.ts +++ b/src/pool-service/IrcPoolClient.ts @@ -16,7 +16,7 @@ import TypedEmitter from "typed-emitter"; const log = new Logger('IrcPoolClient'); -const CONNECTION_TIMEOUT = 20000; +const CONNECTION_TIMEOUT = 40000; const MAX_MISSED_HEARTBEATS = 5; type Events = { @@ -37,6 +37,9 @@ export class IrcPoolClient extends (EventEmitter as unknown as new () => TypedEm this.redis = new Redis(url, { lazyConnect: true, }); + this.redis.on('connecting', () => { + log.debug('Connecting to', url); + }); this.cmdReader = new Redis(url, { lazyConnect: true, }); @@ -62,7 +65,8 @@ export class IrcPoolClient extends (EventEmitter as unknown as new () => TypedEm let isConnected = false; const clientPromise = (async () => { isConnected = (await this.redis.hget(REDIS_IRC_POOL_CONNECTIONS, clientId)) !== null; - const clientState = await IrcClientRedisState.create(this.redis, clientId); + // NOTE: Bandaid solution + const clientState = await IrcClientRedisState.create(this.redis, clientId, !isConnected); return new RedisIrcConnection(this, clientId, clientState); })(); this.connections.set(clientId, clientPromise); @@ -162,8 +166,13 @@ export class IrcPoolClient extends (EventEmitter as unknown as new () => TypedEm public async close() { clearInterval(this.heartbeatInterval); this.shouldRun = false; - this.redis.disconnect(); - this.cmdReader.disconnect(); + // Catch these, because it's quite explosive. + this.redis.quit().catch((ex) => { + log.warn('Failed to quit redis writer', ex); + }); + this.cmdReader.quit().catch((ex) => { + log.warn('Failed to quit redis command reader', ex); + }); } public async handleIncomingCommand() { diff --git a/widget/src/ProvisioningApp.tsx b/widget/src/ProvisioningApp.tsx index b4cb24b72..a08ac8ee7 100644 --- a/widget/src/ProvisioningApp.tsx +++ b/widget/src/ProvisioningApp.tsx @@ -72,7 +72,7 @@ export const ProvisioningApp: React.FC { console.log('Widget API ready'); }); diff --git a/yarn.lock b/yarn.lock index 43a43b13d..b7b29e7f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4249,20 +4249,20 @@ matrix-appservice@^2.0.0: request-promise "^4.2.6" sanitize-html "^2.8.0" -matrix-org-irc@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/matrix-org-irc/-/matrix-org-irc-2.0.0.tgz#b89f932979bd7550c18f551af6dd821b14277c76" - integrity sha512-NFpFX7oa8SF8KBW068o9+k6Gw1LnyMnJflVOXYToxW5NBIDvyYXbuftB1BM3Bsflpv0FgodNLmH35PL/1o0s+w== +matrix-org-irc@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/matrix-org-irc/-/matrix-org-irc-2.0.1.tgz#80cc1ce3abb3e8f240bbc3b4acf1a0fe0f1057e7" + integrity sha512-Qnzx1r5QjTtm63oGUY/XyE3t+1xH43CaC8r3QifkKmBihrYyhHq4bpMsnxNYb558sc2j52pixJ5CUsQ8zMediQ== dependencies: chardet "^1.5.1" iconv-lite "^0.6.3" typed-emitter "^2.1.0" utf-8-validate "^6.0.3" -matrix-widget-api@^1.1.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.3.1.tgz#e38f404c76bb15c113909505c1c1a5b4d781c2f5" - integrity sha512-+rN6vGvnXm+fn0uq9r2KWSL/aPtehD6ObC50jYmUcEfgo8CUpf9eUurmjbRlwZkWq3XHXFuKQBUCI9UzqWg37Q== +matrix-widget-api@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.4.0.tgz#e426ec16a013897f3a4a9c2bff423f54ab0ba745" + integrity sha512-dw0dRylGQzDUoiaY/g5xx1tBbS7aoov31PRtFMAvG58/4uerYllV9Gfou7w+I1aglwB6hihTREzKltVjARWV6A== dependencies: "@types/events" "^3.0.0" events "^3.2.0"