From 3d6664109b7a1717c02be8a48ad354ac637fc27d Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Fri, 22 Nov 2024 13:58:37 +0000
Subject: [PATCH] Send and respect MSC4230 is_animated flag (#28513)

* PoC implementation for is_animated m.image flag

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update MSC reference

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---
 src/@types/matrix-js-sdk.d.ts                |  7 +++++++
 src/ContentMessages.ts                       |  8 +++++++-
 src/components/views/messages/MImageBody.tsx | 13 ++++++++++---
 src/utils/image-media.ts                     | 16 ++--------------
 4 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/src/@types/matrix-js-sdk.d.ts b/src/@types/matrix-js-sdk.d.ts
index dcef9c2eb9..73366f2fee 100644
--- a/src/@types/matrix-js-sdk.d.ts
+++ b/src/@types/matrix-js-sdk.d.ts
@@ -22,6 +22,13 @@ declare module "matrix-js-sdk/src/types" {
         [BLURHASH_FIELD]?: string;
     }
 
+    export interface ImageInfo {
+        /**
+         * @see https://github.com/matrix-org/matrix-spec-proposals/pull/4230
+         */
+        "org.matrix.msc4230.is_animated"?: boolean;
+    }
+
     export interface StateEvents {
         // Jitsi-backed video room state events
         [JitsiCallMemberEventType]: JitsiCallMemberContent;
diff --git a/src/ContentMessages.ts b/src/ContentMessages.ts
index 895e168f3b..344a2f112c 100644
--- a/src/ContentMessages.ts
+++ b/src/ContentMessages.ts
@@ -56,6 +56,7 @@ import { createThumbnail } from "./utils/image-media";
 import { attachMentions, attachRelation } from "./components/views/rooms/SendMessageComposer";
 import { doMaybeLocalRoomAction } from "./utils/local-room";
 import { SdkContextClass } from "./contexts/SDKContext";
+import { blobIsAnimated } from "./utils/Image.ts";
 
 // scraped out of a macOS hidpi (5660ppm) screenshot png
 //                  5669 px (x-axis)      , 5669 px (y-axis)      , per metre
@@ -150,15 +151,20 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag
         thumbnailType = "image/jpeg";
     }
 
+    // We don't await this immediately so it can happen in the background
+    const isAnimatedPromise = blobIsAnimated(imageFile.type, imageFile);
+
     const imageElement = await loadImageElement(imageFile);
 
     const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
     const imageInfo = result.info;
 
+    imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
+
     // For lesser supported image types, always include the thumbnail even if it is larger
     if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) {
         // we do all sizing checks here because we still rely on thumbnail generation for making a blurhash from.
-        const sizeDifference = imageFile.size - imageInfo.thumbnail_info!.size;
+        const sizeDifference = imageFile.size - imageInfo.thumbnail_info!.size!;
         if (
             // image is small enough already
             imageFile.size <= IMAGE_SIZE_THRESHOLD_THUMBNAIL ||
diff --git a/src/components/views/messages/MImageBody.tsx b/src/components/views/messages/MImageBody.tsx
index d8cac8e28b..7d45e71a4b 100644
--- a/src/components/views/messages/MImageBody.tsx
+++ b/src/components/views/messages/MImageBody.tsx
@@ -275,7 +275,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
         }
 
         const content = this.props.mxEvent.getContent<ImageContent>();
-        let isAnimated = mayBeAnimated(content.info?.mimetype);
+        let isAnimated = content.info?.["org.matrix.msc4230.is_animated"] ?? mayBeAnimated(content.info?.mimetype);
 
         // If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
         // because 1. encryption and 2. we can't ask the server specifically for a non-animated thumbnail.
@@ -298,8 +298,15 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
                 }
 
                 try {
-                    const blob = await this.props.mediaEventHelper!.sourceBlob.value;
-                    if (!(await blobIsAnimated(content.info?.mimetype, blob))) {
+                    // If we didn't receive the MSC4230 is_animated flag
+                    // then we need to check if the image is animated by downloading it.
+                    if (
+                        content.info?.["org.matrix.msc4230.is_animated"] === false ||
+                        !(await blobIsAnimated(
+                            content.info?.mimetype,
+                            await this.props.mediaEventHelper!.sourceBlob.value,
+                        ))
+                    ) {
                         isAnimated = false;
                     }
 
diff --git a/src/utils/image-media.ts b/src/utils/image-media.ts
index 5e0fb07678..5c013c7b1a 100644
--- a/src/utils/image-media.ts
+++ b/src/utils/image-media.ts
@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
 Please see LICENSE files in the repository root for full details.
 */
 
-import { EncryptedFile } from "matrix-js-sdk/src/types";
+import { ImageInfo } from "matrix-js-sdk/src/types";
 
 import { BlurhashEncoder } from "../BlurhashEncoder";
 
@@ -15,19 +15,7 @@ type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
 export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448
 
 interface IThumbnail {
-    info: {
-        thumbnail_info?: {
-            w: number;
-            h: number;
-            mimetype: string;
-            size: number;
-        };
-        w: number;
-        h: number;
-        [BLURHASH_FIELD]?: string;
-        thumbnail_url?: string;
-        thumbnail_file?: EncryptedFile;
-    };
+    info: ImageInfo;
     thumbnail: Blob;
 }