From bbd785b1586853d350d6c5373928588b1c8cf599 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 13 Jul 2021 16:57:40 +0100 Subject: [PATCH] Move blurhashing into a Worker and use OffscreenCanvas where possible for thumbnailing --- src/BlurhashEncoder.ts | 59 +++++++++++++++++++++ src/ContentMessages.tsx | 97 +++++++++++++++++++--------------- src/workers/blurhash.worker.ts | 38 +++++++++++++ 3 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 src/BlurhashEncoder.ts create mode 100644 src/workers/blurhash.worker.ts diff --git a/src/BlurhashEncoder.ts b/src/BlurhashEncoder.ts new file mode 100644 index 0000000000..a42c29dfa7 --- /dev/null +++ b/src/BlurhashEncoder.ts @@ -0,0 +1,59 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { defer, IDeferred } from "matrix-js-sdk/src/utils"; + +import BlurhashWorker from "./workers/blurhash.worker.ts"; + +interface IBlurhashWorkerResponse { + seq: number; + blurhash: string; +} + +export class BlurhashEncoder { + private static internalInstance = new BlurhashEncoder(); + + public static get instance(): BlurhashEncoder { + return BlurhashEncoder.internalInstance; + } + + private readonly worker: Worker; + private seq = 0; + private pendingDeferredMap = new Map>(); + + constructor() { + this.worker = new BlurhashWorker(); + this.worker.onmessage = this.onMessage; + } + + private onMessage = (ev: MessageEvent) => { + const { seq, blurhash } = ev.data; + const deferred = this.pendingDeferredMap.get(seq); + if (deferred) { + this.pendingDeferredMap.delete(seq); + deferred.resolve(blurhash); + } + }; + + public getBlurhash(imageData: ImageData): Promise { + const seq = this.seq++; + const deferred = defer(); + this.pendingDeferredMap.set(seq, deferred); + this.worker.postMessage({ seq, imageData }); + return deferred.promise; + } +} + diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index b752886b8a..335784c65a 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -17,7 +17,6 @@ limitations under the License. */ import React from "react"; -import { encode } from "blurhash"; import { MatrixClient } from "matrix-js-sdk/src/client"; import dis from './dispatcher/dispatcher'; @@ -28,7 +27,6 @@ import RoomViewStore from './stores/RoomViewStore'; import encrypt from "browser-encrypt-attachment"; import extractPngChunks from "png-chunks-extract"; import Spinner from "./components/views/elements/Spinner"; - import { Action } from "./dispatcher/actions"; import CountlyAnalytics from "./CountlyAnalytics"; import { @@ -40,6 +38,7 @@ import { } from "./dispatcher/payloads/UploadPayload"; import { IUpload } from "./models/IUpload"; import { IAbortablePromise, IImageInfo } from "matrix-js-sdk/src/@types/partials"; +import { BlurhashEncoder } from "./BlurhashEncoder"; const MAX_WIDTH = 800; const MAX_HEIGHT = 600; @@ -103,55 +102,67 @@ interface IThumbnail { * @return {Promise} A promise that resolves with an object with an info key * and a thumbnail key. */ -function createThumbnail( +async function createThumbnail( element: ThumbnailableElement, inputWidth: number, inputHeight: number, mimeType: string, ): Promise { - return new Promise((resolve) => { - let targetWidth = inputWidth; - let targetHeight = inputHeight; - if (targetHeight > MAX_HEIGHT) { - targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); - targetHeight = MAX_HEIGHT; - } - if (targetWidth > MAX_WIDTH) { - targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); - targetWidth = MAX_WIDTH; - } + let targetWidth = inputWidth; + let targetHeight = inputHeight; + if (targetHeight > MAX_HEIGHT) { + targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); + targetHeight = MAX_HEIGHT; + } + if (targetWidth > MAX_WIDTH) { + targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); + targetWidth = MAX_WIDTH; + } - const canvas = document.createElement("canvas"); + let canvas: HTMLCanvasElement | OffscreenCanvas; + if (window.OffscreenCanvas) { + canvas = new window.OffscreenCanvas(targetWidth, targetHeight); + } else { + canvas = document.createElement("canvas"); canvas.width = targetWidth; canvas.height = targetHeight; - const context = canvas.getContext("2d"); - context.drawImage(element, 0, 0, targetWidth, targetHeight); - const imageData = context.getImageData(0, 0, targetWidth, targetHeight); - const blurhash = encode( - imageData.data, - imageData.width, - imageData.height, - // use 4 components on the longer dimension, if square then both - imageData.width >= imageData.height ? 4 : 3, - imageData.height >= imageData.width ? 4 : 3, - ); - canvas.toBlob(function(thumbnail) { - resolve({ - info: { - thumbnail_info: { - w: targetWidth, - h: targetHeight, - mimetype: thumbnail.type, - size: thumbnail.size, - }, - w: inputWidth, - h: inputHeight, - [BLURHASH_FIELD]: blurhash, - }, - thumbnail, - }); - }, mimeType); - }); + } + + const context = canvas.getContext("2d"); + context.drawImage(element, 0, 0, targetWidth, targetHeight); + + let thumbnailPromise: Promise; + + if (window.OffscreenCanvas) { + thumbnailPromise = (canvas as OffscreenCanvas).convertToBlob({ type: mimeType }); + } else { + thumbnailPromise = new Promise(resolve => (canvas as HTMLCanvasElement).toBlob(resolve, mimeType)); + } + + const imageData = context.getImageData(0, 0, targetWidth, targetHeight); + + const [ + thumbnail, + blurhash, + ] = await Promise.all([ + thumbnailPromise, + BlurhashEncoder.instance.getBlurhash(imageData), + ]); + + return { + info: { + thumbnail_info: { + w: targetWidth, + h: targetHeight, + mimetype: thumbnail.type, + size: thumbnail.size, + }, + w: inputWidth, + h: inputHeight, + [BLURHASH_FIELD]: blurhash, + }, + thumbnail, + }; } /** diff --git a/src/workers/blurhash.worker.ts b/src/workers/blurhash.worker.ts new file mode 100644 index 0000000000..031cc67c90 --- /dev/null +++ b/src/workers/blurhash.worker.ts @@ -0,0 +1,38 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { encode } from "blurhash"; + +const ctx: Worker = self as any; + +interface IBlurhashWorkerRequest { + seq: number; + imageData: ImageData; +} + +ctx.addEventListener("message", (event: MessageEvent): void => { + const { seq, imageData } = event.data; + const blurhash = encode( + imageData.data, + imageData.width, + imageData.height, + // use 4 components on the longer dimension, if square then both + imageData.width >= imageData.height ? 4 : 3, + imageData.height >= imageData.width ? 4 : 3, + ); + + ctx.postMessage({ seq, blurhash }); +});