/* Copyright 2024 New Vector Ltd. Copyright 2022 The Matrix.org Foundation C.I.C. SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only Please see LICENSE files in the repository root for full details. */ import { mocked } from "jest-mock"; import { screen } from "jest-matrix-react"; import userEvent from "@testing-library/user-event"; import { MatrixClient, MatrixEvent, MatrixEventEvent, Room } from "matrix-js-sdk/src/matrix"; import { defer } from "matrix-js-sdk/src/utils"; import { Playback, PlaybackState } from "../../../../src/audio/Playback"; import { PlaybackManager } from "../../../../src/audio/PlaybackManager"; import { SdkContextClass } from "../../../../src/contexts/SDKContext"; import { MediaEventHelper } from "../../../../src/utils/MediaEventHelper"; import { VoiceBroadcastInfoState, VoiceBroadcastLiveness, VoiceBroadcastPlayback, VoiceBroadcastPlaybackEvent, VoiceBroadcastPlaybackState, VoiceBroadcastRecording, } from "../../../../src/voice-broadcast"; import { filterConsole, flushPromises, flushPromisesWithFakeTimers, stubClient, waitEnoughCyclesForModal, } from "../../../test-utils"; import { createTestPlayback } from "../../../test-utils/audio"; import { mkVoiceBroadcastChunkEvent, mkVoiceBroadcastInfoStateEvent } from "../utils/test-utils"; import { LazyValue } from "../../../../src/utils/LazyValue"; jest.mock("../../../../src/utils/MediaEventHelper", () => ({ MediaEventHelper: jest.fn(), })); describe("VoiceBroadcastPlayback", () => { const userId = "@user:example.com"; let deviceId: string; const roomId = "!room:example.com"; let room: Room; let client: MatrixClient; let infoEvent: MatrixEvent; let playback: VoiceBroadcastPlayback; let onStateChanged: (state: VoiceBroadcastPlaybackState) => void; let chunk1Event: MatrixEvent; let deplayedChunk1Event: MatrixEvent; let chunk2Event: MatrixEvent; let chunk2BEvent: MatrixEvent; let chunk3Event: MatrixEvent; const chunk1Length = 2300; const chunk2Length = 4200; const chunk3Length = 6900; const chunk1Data = new ArrayBuffer(2); const chunk2Data = new ArrayBuffer(3); const chunk3Data = new ArrayBuffer(3); let delayedChunk1Helper: MediaEventHelper; let chunk1Helper: MediaEventHelper; let chunk2Helper: MediaEventHelper; let chunk3Helper: MediaEventHelper; let chunk1Playback: Playback; let chunk2Playback: Playback; let chunk3Playback: Playback; let middleOfSecondChunk!: number; let middleOfThirdChunk!: number; const queryConfirmListeningDialog = () => { return screen.queryByText( "If you start listening to this live broadcast, your current live broadcast recording will be ended.", ); }; const itShouldSetTheStateTo = (state: VoiceBroadcastPlaybackState) => { it(`should set the state to ${state}`, () => { expect(playback.getState()).toBe(state); }); }; const itShouldEmitAStateChangedEvent = (state: VoiceBroadcastPlaybackState) => { it(`should emit a ${state} state changed event`, () => { expect(mocked(onStateChanged)).toHaveBeenCalledWith(state, playback); }); }; const itShouldHaveLiveness = (liveness: VoiceBroadcastLiveness): void => { it(`should have liveness ${liveness}`, () => { expect(playback.getLiveness()).toBe(liveness); }); }; const startPlayback = () => { beforeEach(() => { playback.start(); }); }; const pausePlayback = () => { beforeEach(() => { playback.pause(); }); }; const stopPlayback = () => { beforeEach(() => { playback.stop(); }); }; const mkChunkHelper = (data: ArrayBuffer): MediaEventHelper => { return { sourceBlob: { cachedValue: new Blob(), done: false, value: { // @ts-ignore arrayBuffer: jest.fn().mockResolvedValue(data), }, }, }; }; const mkDeplayedChunkHelper = (data: ArrayBuffer): MediaEventHelper => { const deferred = defer>(); setTimeout(() => { deferred.resolve({ // @ts-ignore arrayBuffer: jest.fn().mockResolvedValue(data), }); }, 7500); return { sourceBlob: { cachedValue: new Blob(), done: false, // @ts-ignore value: deferred.promise, }, }; }; const simulateFirstChunkArrived = async (): Promise => { jest.advanceTimersByTime(10000); await flushPromisesWithFakeTimers(); }; const mkInfoEvent = (state: VoiceBroadcastInfoState) => { return mkVoiceBroadcastInfoStateEvent(roomId, state, userId, deviceId); }; const mkPlayback = async (fakeTimers = false): Promise => { const playback = new VoiceBroadcastPlayback( infoEvent, client, SdkContextClass.instance.voiceBroadcastRecordingsStore, ); jest.spyOn(playback, "removeAllListeners"); jest.spyOn(playback, "destroy"); playback.on(VoiceBroadcastPlaybackEvent.StateChanged, onStateChanged); if (fakeTimers) { await flushPromisesWithFakeTimers(); } else { await flushPromises(); } return playback; }; const setUpChunkEvents = (chunkEvents: MatrixEvent[]) => { mocked(client.relations).mockResolvedValueOnce({ events: chunkEvents, }); }; const createChunkEvents = () => { chunk1Event = mkVoiceBroadcastChunkEvent(infoEvent.getId()!, userId, roomId, chunk1Length, 1); deplayedChunk1Event = mkVoiceBroadcastChunkEvent(infoEvent.getId()!, userId, roomId, chunk1Length, 1); chunk2Event = mkVoiceBroadcastChunkEvent(infoEvent.getId()!, userId, roomId, chunk2Length, 2); chunk2Event.setTxnId("tx-id-1"); chunk2BEvent = mkVoiceBroadcastChunkEvent(infoEvent.getId()!, userId, roomId, chunk2Length, 2); chunk2BEvent.setTxnId("tx-id-1"); chunk3Event = mkVoiceBroadcastChunkEvent(infoEvent.getId()!, userId, roomId, chunk3Length, 3); chunk1Helper = mkChunkHelper(chunk1Data); delayedChunk1Helper = mkDeplayedChunkHelper(chunk1Data); chunk2Helper = mkChunkHelper(chunk2Data); chunk3Helper = mkChunkHelper(chunk3Data); chunk1Playback = createTestPlayback(); chunk2Playback = createTestPlayback(); chunk3Playback = createTestPlayback(); middleOfSecondChunk = (chunk1Length + chunk2Length / 2) / 1000; middleOfThirdChunk = (chunk1Length + chunk2Length + chunk3Length / 2) / 1000; jest.spyOn(PlaybackManager.instance, "createPlaybackInstance").mockImplementation( (buffer: ArrayBuffer, _waveForm?: number[]) => { if (buffer === chunk1Data) return chunk1Playback; if (buffer === chunk2Data) return chunk2Playback; if (buffer === chunk3Data) return chunk3Playback; throw new Error("unexpected buffer"); }, ); mocked(MediaEventHelper).mockImplementation((event: MatrixEvent): any => { if (event === chunk1Event) return chunk1Helper; if (event === deplayedChunk1Event) return delayedChunk1Helper; if (event === chunk2Event) return chunk2Helper; if (event === chunk3Event) return chunk3Helper; }); }; filterConsole( // expected for some tests "Unable to load broadcast playback", ); beforeEach(() => { client = stubClient(); deviceId = client.getDeviceId() || ""; room = new Room(roomId, client, client.getSafeUserId()); mocked(client.getRoom).mockImplementation((roomId: string): Room | null => { if (roomId === room.roomId) return room; return null; }); onStateChanged = jest.fn(); }); afterEach(async () => { SdkContextClass.instance.voiceBroadcastPlaybacksStore.getCurrent()?.stop(); SdkContextClass.instance.voiceBroadcastPlaybacksStore.clearCurrent(); await SdkContextClass.instance.voiceBroadcastRecordingsStore.getCurrent()?.stop(); SdkContextClass.instance.voiceBroadcastRecordingsStore.clearCurrent(); playback.destroy(); }); describe(`when there is a ${VoiceBroadcastInfoState.Resumed} broadcast without chunks yet`, () => { beforeEach(async () => { infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Resumed); createChunkEvents(); room.addLiveEvents([infoEvent]); playback = await mkPlayback(); }); describe("and calling start", () => { startPlayback(); itShouldHaveLiveness("live"); it("should be in buffering state", () => { expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Buffering); }); it("should have duration 0", () => { expect(playback.durationSeconds).toBe(0); }); it("should be at time 0", () => { expect(playback.timeSeconds).toBe(0); }); describe("and calling stop", () => { stopPlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); describe("and calling pause", () => { pausePlayback(); // stopped voice broadcasts cannot be paused itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); }); }); describe("and calling pause", () => { pausePlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused); }); describe("and receiving the first chunk", () => { beforeEach(() => { room.relations.aggregateChildEvent(chunk1Event); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); itShouldHaveLiveness("live"); it("should update the duration", () => { expect(playback.durationSeconds).toBe(2.3); }); it("should play the first chunk", () => { expect(chunk1Playback.play).toHaveBeenCalled(); }); }); describe("and receiving the first undecryptable chunk", () => { beforeEach(() => { jest.spyOn(chunk1Event, "isDecryptionFailure").mockReturnValue(true); room.relations.aggregateChildEvent(chunk1Event); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Error); it("should not update the duration", () => { expect(playback.durationSeconds).toBe(0); }); describe("and the chunk is decrypted", () => { beforeEach(() => { mocked(chunk1Event.isDecryptionFailure).mockReturnValue(false); chunk1Event.emit(MatrixEventEvent.Decrypted, chunk1Event); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused); it("should not update the duration", () => { expect(playback.durationSeconds).toBe(2.3); }); }); }); }); }); describe(`when there is a ${VoiceBroadcastInfoState.Resumed} voice broadcast with some chunks`, () => { beforeEach(async () => { mocked(client.relations).mockResolvedValueOnce({ events: [] }); infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Resumed); createChunkEvents(); setUpChunkEvents([chunk2Event, chunk1Event]); room.addLiveEvents([infoEvent, chunk1Event, chunk2Event]); room.relations.aggregateChildEvent(chunk2Event); room.relations.aggregateChildEvent(chunk1Event); playback = await mkPlayback(); }); it("durationSeconds should have the length of the known chunks", () => { expect(playback.durationSeconds).toEqual(6.5); }); describe("and starting a playback with a broken chunk", () => { beforeEach(async () => { mocked(chunk2Playback.prepare).mockRejectedValue("Error decoding chunk"); await playback.start(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Error); it("start() should keep it in the error state)", async () => { await playback.start(); expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Error); }); it("stop() should keep it in the error state)", () => { playback.stop(); expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Error); }); it("toggle() should keep it in the error state)", async () => { await playback.toggle(); expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Error); }); it("pause() should keep it in the error state)", () => { playback.pause(); expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Error); }); }); describe("and an event with the same transaction Id occurs", () => { beforeEach(() => { room.addLiveEvents([chunk2BEvent]); room.relations.aggregateChildEvent(chunk2BEvent); }); it("durationSeconds should not change", () => { expect(playback.durationSeconds).toEqual(6.5); }); }); describe("and calling start", () => { startPlayback(); it("should play the last chunk", () => { expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Playing); // assert that the last chunk is played first expect(chunk2Playback.play).toHaveBeenCalled(); expect(chunk1Playback.play).not.toHaveBeenCalled(); }); describe( "and receiving a stop info event with last_chunk_sequence = 2 and " + "the playback of the last available chunk ends", () => { beforeEach(() => { const stoppedEvent = mkVoiceBroadcastInfoStateEvent( roomId, VoiceBroadcastInfoState.Stopped, client.getSafeUserId(), client.deviceId!, infoEvent, 2, ); room.addLiveEvents([stoppedEvent]); room.relations.aggregateChildEvent(stoppedEvent); chunk2Playback.emit(PlaybackState.Stopped); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); }, ); describe( "and receiving a stop info event with last_chunk_sequence = 3 and " + "the playback of the last available chunk ends", () => { beforeEach(() => { const stoppedEvent = mkVoiceBroadcastInfoStateEvent( roomId, VoiceBroadcastInfoState.Stopped, client.getSafeUserId(), client.deviceId!, infoEvent, 3, ); room.addLiveEvents([stoppedEvent]); room.relations.aggregateChildEvent(stoppedEvent); chunk2Playback.emit(PlaybackState.Stopped); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering); describe("and the next chunk arrives", () => { beforeEach(() => { room.addLiveEvents([chunk3Event]); room.relations.aggregateChildEvent(chunk3Event); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); it("should play the next chunk", () => { expect(chunk3Playback.play).toHaveBeenCalled(); }); }); }, ); describe("and the info event is deleted", () => { beforeEach(() => { infoEvent.makeRedacted(new MatrixEvent({}), room); }); it("should stop and destroy the playback", () => { expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Stopped); expect(playback.destroy).toHaveBeenCalled(); }); }); }); describe("and currently recording a broadcast", () => { let recording: VoiceBroadcastRecording; beforeEach(async () => { recording = new VoiceBroadcastRecording( mkVoiceBroadcastInfoStateEvent( roomId, VoiceBroadcastInfoState.Started, client.getSafeUserId(), client.deviceId, ), client, ); jest.spyOn(recording, "stop"); SdkContextClass.instance.voiceBroadcastRecordingsStore.setCurrent(recording); playback.start(); await waitEnoughCyclesForModal(); }); it("should display a confirm modal", () => { expect(queryConfirmListeningDialog()).toBeInTheDocument(); }); describe("when confirming the dialog", () => { beforeEach(async () => { await userEvent.click(screen.getByText("Yes, end my recording")); }); it("should stop the recording", () => { expect(recording.stop).toHaveBeenCalled(); expect(SdkContextClass.instance.voiceBroadcastRecordingsStore.getCurrent()).toBeNull(); }); it("should not start the playback", () => { expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Playing); }); }); describe("when not confirming the dialog", () => { beforeEach(async () => { await userEvent.click(screen.getByText("No")); }); it("should not stop the recording", () => { expect(recording.stop).not.toHaveBeenCalled(); }); it("should start the playback", () => { expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Stopped); }); }); }); }); describe("when there is a stopped voice broadcast", () => { beforeEach(async () => { jest.useFakeTimers(); infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Stopped); createChunkEvents(); // use delayed first chunk here to simulate loading time setUpChunkEvents([chunk2Event, deplayedChunk1Event, chunk3Event]); room.addLiveEvents([infoEvent, deplayedChunk1Event, chunk2Event, chunk3Event]); playback = await mkPlayback(true); }); afterEach(() => { jest.useRealTimers(); }); it("should expose the info event", () => { expect(playback.infoEvent).toBe(infoEvent); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); describe("and calling start", () => { startPlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering); describe("and the first chunk data has been loaded", () => { beforeEach(async () => { await simulateFirstChunkArrived(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); it("should play the chunks beginning with the first one", () => { // assert that the first chunk is being played expect(chunk1Playback.play).toHaveBeenCalled(); expect(chunk2Playback.play).not.toHaveBeenCalled(); }); describe("and calling start again", () => { it("should not play the first chunk a second time", () => { expect(chunk1Playback.play).toHaveBeenCalledTimes(1); }); }); describe("and the chunk playback progresses", () => { beforeEach(() => { chunk1Playback.clockInfo.liveData.update([11]); }); it("should update the time", () => { expect(playback.timeSeconds).toBe(11); }); }); describe("and the chunk playback progresses across the actual time", () => { // This can be the case if the meta data is out of sync with the actual audio data. beforeEach(() => { chunk1Playback.clockInfo.liveData.update([15]); }); it("should update the time", () => { expect(playback.timeSeconds).toBe(15); expect(playback.timeLeftSeconds).toBe(0); }); }); describe("and skipping to the middle of the second chunk", () => { const middleOfSecondChunk = (chunk1Length + chunk2Length / 2) / 1000; beforeEach(async () => { await playback.skipTo(middleOfSecondChunk); }); it("should play the second chunk", () => { expect(chunk1Playback.stop).toHaveBeenCalled(); expect(chunk1Playback.destroy).toHaveBeenCalled(); expect(chunk2Playback.play).toHaveBeenCalled(); }); it("should update the time", () => { expect(playback.timeSeconds).toBe(middleOfSecondChunk); }); describe("and skipping to the start", () => { beforeEach(async () => { await playback.skipTo(0); }); it("should play the first chunk", () => { expect(chunk2Playback.stop).toHaveBeenCalled(); expect(chunk2Playback.destroy).toHaveBeenCalled(); expect(chunk1Playback.play).toHaveBeenCalled(); }); it("should update the time", () => { expect(playback.timeSeconds).toBe(0); }); }); }); describe("and skipping multiple times", () => { beforeEach(async () => { return Promise.all([ playback.skipTo(middleOfSecondChunk), playback.skipTo(middleOfThirdChunk), playback.skipTo(0), ]); }); it("should only skip to the first and last position", () => { expect(chunk1Playback.stop).toHaveBeenCalled(); expect(chunk1Playback.destroy).toHaveBeenCalled(); expect(chunk2Playback.play).toHaveBeenCalled(); expect(chunk3Playback.play).not.toHaveBeenCalled(); expect(chunk2Playback.stop).toHaveBeenCalled(); expect(chunk2Playback.destroy).toHaveBeenCalled(); expect(chunk1Playback.play).toHaveBeenCalled(); }); }); describe("and the first chunk ends", () => { beforeEach(() => { chunk1Playback.emit(PlaybackState.Stopped); }); it("should play until the end", () => { // assert first chunk was unloaded expect(chunk1Playback.destroy).toHaveBeenCalled(); // assert that the second chunk is being played expect(chunk2Playback.play).toHaveBeenCalled(); // simulate end of second and third chunk chunk2Playback.emit(PlaybackState.Stopped); chunk3Playback.emit(PlaybackState.Stopped); // assert that the entire playback is now in stopped state expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Stopped); }); }); describe("and calling pause", () => { pausePlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused); itShouldEmitAStateChangedEvent(VoiceBroadcastPlaybackState.Paused); }); describe("and calling stop", () => { stopPlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); it("should stop the playback", () => { expect(chunk1Playback.stop).toHaveBeenCalled(); }); describe("and skipping to somewhere in the middle of the first chunk", () => { beforeEach(async () => { mocked(chunk1Playback.play).mockClear(); await playback.skipTo(1); }); it("should not start the playback", () => { expect(chunk1Playback.play).not.toHaveBeenCalled(); }); }); }); describe("and calling destroy", () => { beforeEach(() => { playback.destroy(); }); it("should call removeAllListeners", () => { expect(playback.removeAllListeners).toHaveBeenCalled(); }); it("should call destroy on the playbacks", () => { expect(chunk1Playback.destroy).toHaveBeenCalled(); expect(chunk2Playback.destroy).toHaveBeenCalled(); }); }); }); }); describe("and calling toggle for the first time", () => { beforeEach(async () => { playback.toggle(); await simulateFirstChunkArrived(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); describe("and calling toggle a second time", () => { beforeEach(async () => { await playback.toggle(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Paused); describe("and calling toggle a third time", () => { beforeEach(async () => { await playback.toggle(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); }); }); }); describe("and calling stop", () => { stopPlayback(); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped); describe("and calling toggle", () => { beforeEach(async () => { mocked(onStateChanged).mockReset(); playback.toggle(); await simulateFirstChunkArrived(); }); itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing); itShouldEmitAStateChangedEvent(VoiceBroadcastPlaybackState.Playing); }); }); }); });