element-web-Github/src/ContentMessages.ts

/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
Copyright 2019 New Vector Ltd
Copyright 2015, 2016 OpenMarket Ltd

SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
Please see LICENSE files in the repository root for full details.
*/

import {
    MatrixClient,
    MsgType,
    HTTPError,
    IEventRelation,
    ISendEventResponse,
    MatrixEvent,
    UploadOpts,
    UploadProgress,
    THREAD_RELATION_TYPE,
} from "matrix-js-sdk/src/matrix";
import {
    ImageInfo,
    AudioInfo,
    VideoInfo,
    EncryptedFile,
    MediaEventContent,
    MediaEventInfo,
} from "matrix-js-sdk/src/types";
import encrypt from "matrix-encrypt-attachment";
import extractPngChunks from "png-chunks-extract";
import { logger } from "matrix-js-sdk/src/logger";
import { removeElement } from "matrix-js-sdk/src/utils";

import dis from "./dispatcher/dispatcher";
import { _t } from "./languageHandler";
import Modal from "./Modal";
import Spinner from "./components/views/elements/Spinner";
import { Action } from "./dispatcher/actions";
import {
    UploadCanceledPayload,
    UploadErrorPayload,
    UploadFinishedPayload,
    UploadProgressPayload,
    UploadStartedPayload,
} from "./dispatcher/payloads/UploadPayload";
import { RoomUpload } from "./models/RoomUpload";
import SettingsStore from "./settings/SettingsStore";
import { decorateStartSendingTime, sendRoundTripMetric } from "./sendTimePerformanceMetrics";
import { TimelineRenderingType } from "./contexts/RoomContext";
import { addReplyToMessageContent } from "./utils/Reply";
import ErrorDialog from "./components/views/dialogs/ErrorDialog";
import UploadFailureDialog from "./components/views/dialogs/UploadFailureDialog";
import UploadConfirmDialog from "./components/views/dialogs/UploadConfirmDialog";
import { createThumbnail } from "./utils/image-media";
import { attachMentions, attachRelation } from "./components/views/rooms/SendMessageComposer";
import { doMaybeLocalRoomAction } from "./utils/local-room";
import { SdkContextClass } from "./contexts/SDKContext";

// scraped out of a macOS hidpi (5660ppm) screenshot png
//                  5669 px (x-axis)      , 5669 px (y-axis)      , per metre
const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01];

export class UploadCanceledError extends Error {}

interface IMediaConfig {
    "m.upload.size"?: number;
}

/**
 * Load a file into a newly created image element.
 *
 * @param {File} imageFile The file to load in an image element.
 * @return {Promise} A promise that resolves with the html image element.
 */
async function loadImageElement(imageFile: File): Promise<{
    width: number;
    height: number;
    img: HTMLImageElement;
}> {
    // Load the file into an html element
    const img = new Image();
    const objectUrl = URL.createObjectURL(imageFile);
    const imgPromise = new Promise((resolve, reject) => {
        img.onload = function (): void {
            URL.revokeObjectURL(objectUrl);
            resolve(img);
        };
        img.onerror = function (e): void {
            reject(e);
        };
    });
    img.src = objectUrl;

    // check for hi-dpi PNGs and fudge display resolution as needed.
    // this is mainly needed for macOS screencaps
    let parsePromise = Promise.resolve(false);
    if (imageFile.type === "image/png") {
        // in practice macOS happens to order the chunks so they fall in
        // the first 0x1000 bytes (thanks to a massive ICC header).
        // Thus we could slice the file down to only sniff the first 0x1000
        // bytes (but this makes extractPngChunks choke on the corrupt file)
        const headers = imageFile; //.slice(0, 0x1000);
        parsePromise = readFileAsArrayBuffer(headers)
            .then((arrayBuffer) => {
                const buffer = new Uint8Array(arrayBuffer);
                const chunks = extractPngChunks(buffer);
                for (const chunk of chunks) {
                    if (chunk.name === "pHYs") {
                        if (chunk.data.byteLength !== PHYS_HIDPI.length) return false;
                        return chunk.data.every((val, i) => val === PHYS_HIDPI[i]);
                    }
                }
                return false;
            })
            .catch((e) => {
                console.error("Failed to parse PNG", e);
                return false;
            });
    }

    const [hidpi] = await Promise.all([parsePromise, imgPromise]);
    const width = hidpi ? img.width >> 1 : img.width;
    const height = hidpi ? img.height >> 1 : img.height;
    return { width, height, img };
}

// Minimum size for image files before we generate a thumbnail for them.
const IMAGE_SIZE_THRESHOLD_THUMBNAIL = 1 << 15; // 32KB
// Minimum size improvement for image thumbnails, if both are not met then don't bother uploading thumbnail.
const IMAGE_THUMBNAIL_MIN_REDUCTION_SIZE = 1 << 16; // 1MB
const IMAGE_THUMBNAIL_MIN_REDUCTION_PERCENT = 0.1; // 10%
// We don't apply these thresholds to video thumbnails as a poster image is always useful
// and videos tend to be much larger.

// Image mime types for which to always include a thumbnail for even if it is larger than the input for wider support.
const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];

/**
 * Read the metadata for an image file and create and upload a thumbnail of the image.
 *
 * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
 * @param {String} roomId The ID of the room the image will be uploaded in.
 * @param {File} imageFile The image to read and thumbnail.
 * @return {Promise} A promise that resolves with the attachment info.
 */
async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
    let thumbnailType = "image/png";
    if (imageFile.type === "image/jpeg") {
        thumbnailType = "image/jpeg";
    }

    const imageElement = await loadImageElement(imageFile);

    const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
    const imageInfo = result.info;

    // For lesser supported image types, always include the thumbnail even if it is larger
    if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) {
        // we do all sizing checks here because we still rely on thumbnail generation for making a blurhash from.
        const sizeDifference = imageFile.size - imageInfo.thumbnail_info!.size;
        if (
            // image is small enough already
            imageFile.size <= IMAGE_SIZE_THRESHOLD_THUMBNAIL ||
            // thumbnail is not sufficiently smaller than original
            (sizeDifference <= IMAGE_THUMBNAIL_MIN_REDUCTION_SIZE &&
                sizeDifference <= imageFile.size * IMAGE_THUMBNAIL_MIN_REDUCTION_PERCENT)
        ) {
            delete imageInfo["thumbnail_info"];
            return imageInfo;
        }
    }

    const uploadResult = await uploadFile(matrixClient, roomId, result.thumbnail);

    imageInfo["thumbnail_url"] = uploadResult.url;
    imageInfo["thumbnail_file"] = uploadResult.file;
    return imageInfo;
}

/**
 * Load a file into a newly created audio element and load the metadata
 *
 * @param {File} audioFile The file to load in an audio element.
 * @return {Promise} A promise that resolves with the audio element.
 */
function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
    return new Promise((resolve, reject) => {
        // Load the file into a html element
        const audio = document.createElement("audio");
        audio.preload = "metadata";
        audio.muted = true;

        const reader = new FileReader();

        reader.onload = function (ev): void {
            audio.onloadedmetadata = async function (): Promise<void> {
                resolve(audio);
            };
            audio.onerror = function (e): void {
                reject(e);
            };

            audio.src = ev.target?.result as string;
        };
        reader.onerror = function (e): void {
            reject(e);
        };
        reader.readAsDataURL(audioFile);
    });
}

/**
 * Read the metadata for an audio file.
 *
 * @param {File} audioFile The audio to read.
 * @return {Promise} A promise that resolves with the attachment info.
 */
async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
    const audio = await loadAudioElement(audioFile);
    return { duration: Math.ceil(audio.duration * 1000) };
}

/**
 * Load a file into a newly created video element and pull some strings
 * in an attempt to guarantee the first frame will be showing.
 *
 * @param {File} videoFile The file to load in a video element.
 * @return {Promise} A promise that resolves with the video element.
 */
function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
    return new Promise((resolve, reject) => {
        // Load the file into a html element
        const video = document.createElement("video");
        video.preload = "metadata";
        video.playsInline = true;
        video.muted = true;

        const reader = new FileReader();

        reader.onload = function (ev): void {
            // Wait until we have enough data to thumbnail the first frame.
            video.onloadeddata = async function (): Promise<void> {
                resolve(video);
                video.pause();
            };
            video.onerror = function (e): void {
                reject(e);
            };

            let dataUrl = ev.target?.result as string;
            // Chrome chokes on quicktime but likes mp4, and `file.type` is
            // read only, so do this horrible hack to unbreak quicktime
            if (dataUrl?.startsWith("data:video/quicktime;")) {
                dataUrl = dataUrl.replace("data:video/quicktime;", "data:video/mp4;");
            }

            video.src = dataUrl;
            video.load();
            video.play();
        };
        reader.onerror = function (e): void {
            reject(e);
        };
        reader.readAsDataURL(videoFile);
    });
}

/**
 * Read the metadata for a video file and create and upload a thumbnail of the video.
 *
 * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
 * @param {String} roomId The ID of the room the video will be uploaded to.
 * @param {File} videoFile The video to read and thumbnail.
 * @return {Promise} A promise that resolves with the attachment info.
 */
function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
    const thumbnailType = "image/jpeg";

    const videoInfo: VideoInfo = {};
    return loadVideoElement(videoFile)
        .then((video) => {
            videoInfo.duration = Math.ceil(video.duration * 1000);
            return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
        })
        .then((result) => {
            Object.assign(videoInfo, result.info);
            return uploadFile(matrixClient, roomId, result.thumbnail);
        })
        .then((result) => {
            videoInfo.thumbnail_url = result.url;
            videoInfo.thumbnail_file = result.file;
            return videoInfo;
        });
}

/**
 * Read the file as an ArrayBuffer.
 * @param {File} file The file to read
 * @return {Promise} A promise that resolves with an ArrayBuffer when the file
 *   is read.
 */
function readFileAsArrayBuffer(file: File | Blob): Promise<ArrayBuffer> {
    return new Promise((resolve, reject) => {
        const reader = new FileReader();
        reader.onload = function (e): void {
            resolve(e.target?.result as ArrayBuffer);
        };
        reader.onerror = function (e): void {
            reject(e);
        };
        reader.readAsArrayBuffer(file);
    });
}

/**
 * Upload the file to the content repository.
 * If the room is encrypted then encrypt the file before uploading.
 *
 * @param {MatrixClient} matrixClient The matrix client to upload the file with.
 * @param {String} roomId The ID of the room being uploaded to.
 * @param {File} file The file to upload.
 * @param {Function?} progressHandler optional callback to be called when a chunk of
 *    data is uploaded.
 * @param {AbortController?} controller optional abortController to use for this upload.
 * @return {Promise} A promise that resolves with an object.
 *  If the file is unencrypted then the object will have a "url" key.
 *  If the file is encrypted then the object will have a "file" key.
 */
export async function uploadFile(
    matrixClient: MatrixClient,
    roomId: string,
    file: File | Blob,
    progressHandler?: UploadOpts["progressHandler"],
    controller?: AbortController,
): Promise<{ url?: string; file?: EncryptedFile }> {
    const abortController = controller ?? new AbortController();

    // If the room is encrypted then encrypt the file before uploading it.
    if (matrixClient.isRoomEncrypted(roomId)) {
        // First read the file into memory.
        const data = await readFileAsArrayBuffer(file);
        if (abortController.signal.aborted) throw new UploadCanceledError();

        // Then encrypt the file.
        const encryptResult = await encrypt.encryptAttachment(data);
        if (abortController.signal.aborted) throw new UploadCanceledError();

        // Pass the encrypted data as a Blob to the uploader.
        const blob = new Blob([encryptResult.data]);

        const { content_uri: url } = await matrixClient.uploadContent(blob, {
            progressHandler,
            abortController,
            includeFilename: false,
            type: "application/octet-stream",
        });
        if (abortController.signal.aborted) throw new UploadCanceledError();

        // If the attachment is encrypted then bundle the URL along with the information
        // needed to decrypt the attachment and add it under a file key.
        return {
            file: {
                ...encryptResult.info,
                url,
            } as EncryptedFile,
        };
    } else {
        const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
        if (abortController.signal.aborted) throw new UploadCanceledError();
        // If the attachment isn't encrypted then include the URL directly.
        return { url };
    }
}

export default class ContentMessages {
    private inprogress: RoomUpload[] = [];
    private mediaConfig: IMediaConfig | null = null;

    public sendStickerContentToRoom(
        url: string,
        roomId: string,
        threadId: string | null,
        info: ImageInfo,
        text: string,
        matrixClient: MatrixClient,
    ): Promise<ISendEventResponse> {
        return doMaybeLocalRoomAction(
            roomId,
            (actualRoomId: string) => matrixClient.sendStickerMessage(actualRoomId, threadId, url, info, text),
            matrixClient,
        ).catch((e) => {
            logger.warn(`Failed to send content with URL ${url} to room ${roomId}`, e);
            throw e;
        });
    }

    public getUploadLimit(): number | null {
        if (this.mediaConfig !== null && this.mediaConfig["m.upload.size"] !== undefined) {
            return this.mediaConfig["m.upload.size"];
        } else {
            return null;
        }
    }

    public async sendContentListToRoom(
        files: File[],
        roomId: string,
        relation: IEventRelation | undefined,
        matrixClient: MatrixClient,
        context = TimelineRenderingType.Room,
    ): Promise<void> {
        if (matrixClient.isGuest()) {
            dis.dispatch({ action: "require_registration" });
            return;
        }

        const replyToEvent = SdkContextClass.instance.roomViewStore.getQuotingEvent();
        if (!this.mediaConfig) {
            // hot-path optimization to not flash a spinner if we don't need to
            const modal = Modal.createDialog(Spinner, undefined, "mx_Dialog_spinner");
            await Promise.race([this.ensureMediaConfigFetched(matrixClient), modal.finished]);
            if (!this.mediaConfig) {
                // User cancelled by clicking away on the spinner
                return;
            } else {
                modal.close();
            }
        }

        const tooBigFiles: File[] = [];
        const okFiles: File[] = [];

        for (const file of files) {
            if (this.isFileSizeAcceptable(file)) {
                okFiles.push(file);
            } else {
                tooBigFiles.push(file);
            }
        }

        if (tooBigFiles.length > 0) {
            const { finished } = Modal.createDialog(UploadFailureDialog, {
                badFiles: tooBigFiles,
                totalFiles: files.length,
                contentMessages: this,
            });
            const [shouldContinue] = await finished;
            if (!shouldContinue) return;
        }

        let uploadAll = false;
        // Promise to complete before sending next file into room, used for synchronisation of file-sending
        // to match the order the files were specified in
        let promBefore: Promise<any> = Promise.resolve();
        for (let i = 0; i < okFiles.length; ++i) {
            const file = okFiles[i];
            const loopPromiseBefore = promBefore;

            if (!uploadAll) {
                const { finished } = Modal.createDialog(UploadConfirmDialog, {
                    file,
                    currentIndex: i,
                    totalFiles: okFiles.length,
                });
                const [shouldContinue, shouldUploadAll] = await finished;
                if (!shouldContinue) break;
                if (shouldUploadAll) {
                    uploadAll = true;
                }
            }

            promBefore = doMaybeLocalRoomAction(
                roomId,
                (actualRoomId) =>
                    this.sendContentToRoom(
                        file,
                        actualRoomId,
                        relation,
                        matrixClient,
                        replyToEvent ?? undefined,
                        loopPromiseBefore,
                    ),
                matrixClient,
            );
        }

        if (replyToEvent) {
            // Clear event being replied to
            dis.dispatch({
                action: "reply_to_event",
                event: null,
                context,
            });
        }

        // Focus the correct composer
        dis.dispatch({
            action: Action.FocusSendMessageComposer,
            context,
        });
    }

    public getCurrentUploads(relation?: IEventRelation): RoomUpload[] {
        return this.inprogress.filter((roomUpload) => {
            const noRelation = !relation && !roomUpload.relation;
            const matchingRelation =
                relation &&
                roomUpload.relation &&
                relation.rel_type === roomUpload.relation.rel_type &&
                relation.event_id === roomUpload.relation.event_id;

            return (noRelation || matchingRelation) && !roomUpload.cancelled;
        });
    }

    public cancelUpload(upload: RoomUpload): void {
        upload.abort();
        dis.dispatch<UploadCanceledPayload>({ action: Action.UploadCanceled, upload });
    }

    public async sendContentToRoom(
        file: File,
        roomId: string,
        relation: IEventRelation | undefined,
        matrixClient: MatrixClient,
        replyToEvent: MatrixEvent | undefined,
        promBefore?: Promise<any>,
    ): Promise<void> {
        const fileName = file.name || _t("common|attachment");
        const content: Omit<MediaEventContent, "info"> & { info: Partial<MediaEventInfo> } = {
            body: fileName,
            info: {
                size: file.size,
            },
            msgtype: MsgType.File, // set more specifically later
        };

        // Attach mentions, which really only applies if there's a replyToEvent.
        attachMentions(matrixClient.getSafeUserId(), content, null, replyToEvent);
        attachRelation(content, relation);
        if (replyToEvent) {
            addReplyToMessageContent(content, replyToEvent, {
                includeLegacyFallback: false,
            });
        }

        if (SettingsStore.getValue("Performance.addSendMessageTimingMetadata")) {
            decorateStartSendingTime(content);
        }

        // if we have a mime type for the file, add it to the message metadata
        if (file.type) {
            content.info.mimetype = file.type;
        }

        const upload = new RoomUpload(roomId, fileName, relation, file.size);
        this.inprogress.push(upload);
        dis.dispatch<UploadStartedPayload>({ action: Action.UploadStarted, upload });

        function onProgress(progress: UploadProgress): void {
            upload.onProgress(progress);
            dis.dispatch<UploadProgressPayload>({ action: Action.UploadProgress, upload });
        }

        try {
            if (file.type.startsWith("image/")) {
                content.msgtype = MsgType.Image;
                try {
                    const imageInfo = await infoForImageFile(matrixClient, roomId, file);
                    Object.assign(content.info, imageInfo);
                } catch (e) {
                    if (e instanceof HTTPError) {
                        // re-throw to main upload error handler
                        throw e;
                    }
                    // Otherwise we failed to thumbnail, fall back to uploading an m.file
                    logger.error(e);
                    content.msgtype = MsgType.File;
                }
            } else if (file.type.indexOf("audio/") === 0) {
                content.msgtype = MsgType.Audio;
                try {
                    const audioInfo = await infoForAudioFile(file);
                    Object.assign(content.info, audioInfo);
                } catch (e) {
                    // Failed to process audio file, fall back to uploading an m.file
                    logger.error(e);
                    content.msgtype = MsgType.File;
                }
            } else if (file.type.indexOf("video/") === 0) {
                content.msgtype = MsgType.Video;
                try {
                    const videoInfo = await infoForVideoFile(matrixClient, roomId, file);
                    Object.assign(content.info, videoInfo);
                } catch (e) {
                    // Failed to thumbnail, fall back to uploading an m.file
                    logger.error(e);
                    content.msgtype = MsgType.File;
                }
            } else {
                content.msgtype = MsgType.File;
            }

            if (upload.cancelled) throw new UploadCanceledError();
            const result = await uploadFile(matrixClient, roomId, file, onProgress, upload.abortController);
            content.file = result.file;
            content.url = result.url;

            if (upload.cancelled) throw new UploadCanceledError();
            // Await previous message being sent into the room
            if (promBefore) await promBefore;

            if (upload.cancelled) throw new UploadCanceledError();
            const threadId = relation?.rel_type === THREAD_RELATION_TYPE.name ? relation.event_id : null;

            const response = await matrixClient.sendMessage(roomId, threadId ?? null, content as MediaEventContent);

            if (SettingsStore.getValue("Performance.addSendMessageTimingMetadata")) {
                sendRoundTripMetric(matrixClient, roomId, response.event_id);
            }

            dis.dispatch<UploadFinishedPayload>({ action: Action.UploadFinished, upload });
            dis.dispatch({ action: "message_sent" });
        } catch (error) {
            // 413: File was too big or upset the server in some way:
            // clear the media size limit so we fetch it again next time we try to upload
            if (error instanceof HTTPError && error.httpStatus === 413) {
                this.mediaConfig = null;
            }

            if (!upload.cancelled) {
                let desc = _t("upload_failed_generic", { fileName: upload.fileName });
                if (error instanceof HTTPError && error.httpStatus === 413) {
                    desc = _t("upload_failed_size", {
                        fileName: upload.fileName,
                    });
                }
                Modal.createDialog(ErrorDialog, {
                    title: _t("upload_failed_title"),
                    description: desc,
                });
                dis.dispatch<UploadErrorPayload>({ action: Action.UploadFailed, upload, error });
            }
        } finally {
            removeElement(this.inprogress, (e) => e.promise === upload.promise);
        }
    }

    private isFileSizeAcceptable(file: File): boolean {
        if (
            this.mediaConfig !== null &&
            this.mediaConfig["m.upload.size"] !== undefined &&
            file.size > this.mediaConfig["m.upload.size"]
        ) {
            return false;
        }
        return true;
    }

    private ensureMediaConfigFetched(matrixClient: MatrixClient): Promise<void> {
        if (this.mediaConfig !== null) return Promise.resolve();

        logger.log("[Media Config] Fetching");
        return matrixClient
            .getMediaConfig()
            .then((config) => {
                logger.log("[Media Config] Fetched config:", config);
                return config;
            })
            .catch(() => {
                // Media repo can't or won't report limits, so provide an empty object (no limits).
                logger.log("[Media Config] Could not fetch config, so not limiting uploads.");
                return {};
            })
            .then((config) => {
                this.mediaConfig = config;
            });
    }

    public static sharedInstance(): ContentMessages {
        if (window.mxContentMessages === undefined) {
            window.mxContentMessages = new ContentMessages();
        }
        return window.mxContentMessages;
    }
}