From 13f28e53e1d7e55157e8ae00936b355579004f60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 15 Nov 2016 11:22:39 +0000 Subject: [PATCH] Generate thumbnails when sending m.image and m.video messages. (#555) * Send a thumbnail when sending a m.image * Use the 'thumbnail_file' when displaying encrypted images * Whitespace * Generate thumbnails for m.video * Fix docstring, remove unused vars, use const * Don't change the upload promise behaviour * Polyfill for Canvas.toBlob to support older browsers * Lowercase for integer types in jsdoc --- package.json | 1 + src/ContentMessages.js | 260 +++++++++++++++----- src/components/views/messages/MImageBody.js | 25 +- 3 files changed, 219 insertions(+), 67 deletions(-) diff --git a/package.json b/package.json index 34bb87eb79..7d822fae89 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ }, "dependencies": { "babel-runtime": "^6.11.6", + "blueimp-canvas-to-blob": "^3.5.0", "browser-encrypt-attachment": "^0.1.0", "browser-request": "^0.3.3", "classnames": "^2.1.2", diff --git a/src/ContentMessages.js b/src/ContentMessages.js index 28c28e875e..e2f9086f66 100644 --- a/src/ContentMessages.js +++ b/src/ContentMessages.js @@ -25,22 +25,87 @@ var Modal = require('./Modal'); var encrypt = require("browser-encrypt-attachment"); -function infoForImageFile(imageFile) { - var deferred = q.defer(); +// Polyfill for Canvas.toBlob API using Canvas.toDataURL +require("blueimp-canvas-to-blob"); + +const MAX_WIDTH = 800; +const MAX_HEIGHT = 600; + + +/** + * Create a thumbnail for a image DOM element. + * The image will be smaller than MAX_WIDTH and MAX_HEIGHT. + * The thumbnail will have the same aspect ratio as the original. + * Draws the element into a canvas using CanvasRenderingContext2D.drawImage + * Then calls Canvas.toBlob to get a blob object for the image data. + * + * Since it needs to calculate the dimensions of the source image and the + * thumbnailed image it returns an info object filled out with information + * about the original image and the thumbnail. + * + * @param {HTMLElement} element The element to thumbnail. + * @param {integer} inputWidth The width of the image in the input element. + * @param {integer} inputHeight the width of the image in the input element. + * @param {String} mimeType The mimeType to save the blob as. + * @return {Promise} A promise that resolves with an object with an info key + * and a thumbnail key. + */ +function createThumbnail(element, inputWidth, inputHeight, mimeType) { + const deferred = q.defer(); + + var targetWidth = inputWidth; + var targetHeight = inputHeight; + if (targetHeight > MAX_HEIGHT) { + targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight)); + targetHeight = MAX_HEIGHT; + } + if (targetWidth > MAX_WIDTH) { + targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth)); + targetWidth = MAX_WIDTH; + } + + const canvas = document.createElement("canvas"); + canvas.width = targetWidth; + canvas.height = targetHeight; + canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight); + canvas.toBlob(function(thumbnail) { + deferred.resolve({ + info: { + thumbnail_info: { + w: targetWidth, + h: targetHeight, + mimetype: thumbnail.type, + size: thumbnail.size, + }, + w: inputWidth, + h: inputHeight, + }, + thumbnail: thumbnail + }); + }, mimeType); + + return deferred.promise; +} + +/** + * Load a file into a newly created image element. + * + * @param {File} file The file to load in an image element. + * @return {Promise} A promise that resolves with the html image element. + */ +function loadImageElement(imageFile) { + const deferred = q.defer(); // Load the file into an html element - var img = document.createElement("img"); + const img = document.createElement("img"); - var reader = new FileReader(); + const reader = new FileReader(); reader.onload = function(e) { img.src = e.target.result; - // Once ready, returns its size + // Once ready, create a thumbnail img.onload = function() { - deferred.resolve({ - w: img.width, - h: img.height - }); + deferred.resolve(img); }; img.onerror = function(e) { deferred.reject(e); @@ -54,22 +119,53 @@ function infoForImageFile(imageFile) { return deferred.promise; } -function infoForVideoFile(videoFile) { - var deferred = q.defer(); +/** + * Read the metadata for an image file and create and upload a thumbnail of the image. + * + * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with. + * @param {String} roomId The ID of the room the image will be uploaded in. + * @param {File} The image to read and thumbnail. + * @return {Promise} A promise that resolves with the attachment info. + */ +function infoForImageFile(matrixClient, roomId, imageFile) { + var thumbnailType = "image/png"; + if (imageFile.type == "image/jpeg") { + thumbnailType = "image/jpeg"; + } + + var imageInfo; + return loadImageElement(imageFile).then(function(img) { + return createThumbnail(img, img.width, img.height, thumbnailType); + }).then(function(result) { + imageInfo = result.info; + return uploadFile(matrixClient, roomId, result.thumbnail); + }).then(function(result) { + imageInfo.thumbnail_url = result.url; + imageInfo.thumbnail_file = result.file; + return imageInfo; + }); +} + +/** + * Load a file into a newly created video element. + * + * @param {File} file The file to load in an video element. + * @return {Promise} A promise that resolves with the video image element. + */ +function loadVideoElement(videoFile) { + const deferred = q.defer(); // Load the file into an html element - var video = document.createElement("video"); + const video = document.createElement("video"); - var reader = new FileReader(); + const reader = new FileReader(); reader.onload = function(e) { video.src = e.target.result; // Once ready, returns its size - video.onloadedmetadata = function() { - deferred.resolve({ - w: video.videoWidth, - h: video.videoHeight - }); + // Wait until we have enough data to thumbnail the first frame. + video.onloadeddata = function() { + deferred.resolve(video); }; video.onerror = function(e) { deferred.reject(e); @@ -83,6 +179,30 @@ function infoForVideoFile(videoFile) { return deferred.promise; } +/** + * Read the metadata for a video file and create and upload a thumbnail of the video. + * + * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with. + * @param {String} roomId The ID of the room the video will be uploaded to. + * @param {File} The video to read and thumbnail. + * @return {Promise} A promise that resolves with the attachment info. + */ +function infoForVideoFile(matrixClient, roomId, videoFile) { + const thumbnailType = "image/jpeg"; + + var videoInfo; + return loadVideoElement(videoFile).then(function(video) { + return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType); + }).then(function(result) { + videoInfo = result.info; + return uploadFile(matrixClient, roomId, result.thumbnail); + }).then(function(result) { + videoInfo.thumbnail_url = result.url; + videoInfo.thumbnail_file = result.file; + return videoInfo; + }); +} + /** * Read the file as an ArrayBuffer. * @return {Promise} A promise that resolves with an ArrayBuffer when the file @@ -101,6 +221,48 @@ function readFileAsArrayBuffer(file) { return deferred.promise; } +/** + * Upload the file to the content repository. + * If the room is encrypted then encrypt the file before uploading. + * + * @param {MatrixClient} matrixClient The matrix client to upload the file with. + * @param {String} roomId The ID of the room being uploaded to. + * @param {File} file The file to upload. + * @return {Promise} A promise that resolves with an object. + * If the file is unencrypted then the object will have a "url" key. + * If the file is encrypted then the object will have a "file" key. + */ +function uploadFile(matrixClient, roomId, file) { + if (matrixClient.isRoomEncrypted(roomId)) { + // If the room is encrypted then encrypt the file before uploading it. + // First read the file into memory. + return readFileAsArrayBuffer(file).then(function(data) { + // Then encrypt the file. + return encrypt.encryptAttachment(data); + }).then(function(encryptResult) { + // Record the information needed to decrypt the attachment. + const encryptInfo = encryptResult.info; + // Pass the encrypted data as a Blob to the uploader. + const blob = new Blob([encryptResult.data]); + return matrixClient.uploadContent(blob).then(function(url) { + // If the attachment is encrypted then bundle the URL along + // with the information needed to decrypt the attachment and + // add it under a file key. + encryptInfo.url = url; + if (file.type) { + encryptInfo.mimetype = file.type; + } + return {"file": encryptInfo}; + }); + }); + } else { + return matrixClient.uploadContent(file).then(function(url) { + // If the attachment isn't encrypted then include the URL directly. + return {"url": url}; + }); + } +} + class ContentMessages { constructor() { @@ -109,7 +271,7 @@ class ContentMessages { } sendContentToRoom(file, roomId, matrixClient) { - var content = { + const content = { body: file.name, info: { size: file.size, @@ -121,13 +283,14 @@ class ContentMessages { content.info.mimetype = file.type; } - var def = q.defer(); + const def = q.defer(); if (file.type.indexOf('image/') == 0) { content.msgtype = 'm.image'; - infoForImageFile(file).then(imageInfo=>{ + infoForImageFile(matrixClient, roomId, file).then(imageInfo=>{ extend(content.info, imageInfo); def.resolve(); }, error=>{ + console.error(error); content.msgtype = 'm.file'; def.resolve(); }); @@ -136,7 +299,7 @@ class ContentMessages { def.resolve(); } else if (file.type.indexOf('video/') == 0) { content.msgtype = 'm.video'; - infoForVideoFile(file).then(videoInfo=>{ + infoForVideoFile(matrixClient, roomId, file).then(videoInfo=>{ extend(content.info, videoInfo); def.resolve(); }, error=>{ @@ -148,35 +311,23 @@ class ContentMessages { def.resolve(); } - var upload = { + const upload = { fileName: file.name, roomId: roomId, total: 0, - loaded: 0 + loaded: 0, }; this.inprogress.push(upload); dis.dispatch({action: 'upload_started'}); - var encryptInfo = null; var error; - var self = this; return def.promise.then(function() { - if (matrixClient.isRoomEncrypted(roomId)) { - // If the room is encrypted then encrypt the file before uploading it. - // First read the file into memory. - upload.promise = readFileAsArrayBuffer(file).then(function(data) { - // Then encrypt the file. - return encrypt.encryptAttachment(data); - }).then(function(encryptResult) { - // Record the information needed to decrypt the attachment. - encryptInfo = encryptResult.info; - // Pass the encrypted data as a Blob to the uploader. - var blob = new Blob([encryptResult.data]); - return matrixClient.uploadContent(blob); - }); - } else { - upload.promise = matrixClient.uploadContent(file); - } + upload.promise = uploadFile( + matrixClient, roomId, file + ).then(function(result) { + content.file = result.file; + content.url = result.url; + }); return upload.promise; }).progress(function(ev) { if (ev) { @@ -185,19 +336,6 @@ class ContentMessages { dis.dispatch({action: 'upload_progress', upload: upload}); } }).then(function(url) { - if (encryptInfo === null) { - // If the attachment isn't encrypted then include the URL directly. - content.url = url; - } else { - // If the attachment is encrypted then bundle the URL along - // with the information needed to decrypt the attachment and - // add it under a file key. - encryptInfo.url = url; - if (file.type) { - encryptInfo.mimetype = file.type; - } - content.file = encryptInfo; - } return matrixClient.sendMessage(roomId, content); }, function(err) { error = err; @@ -212,12 +350,12 @@ class ContentMessages { description: desc }); } - }).finally(function() { - var inprogressKeys = Object.keys(self.inprogress); - for (var i = 0; i < self.inprogress.length; ++i) { + }).finally(() => { + const inprogressKeys = Object.keys(this.inprogress); + for (var i = 0; i < this.inprogress.length; ++i) { var k = inprogressKeys[i]; - if (self.inprogress[k].promise === upload.promise) { - self.inprogress.splice(k, 1); + if (this.inprogress[k].promise === upload.promise) { + this.inprogress.splice(k, 1); break; } } @@ -235,7 +373,7 @@ class ContentMessages { } cancelUpload(promise) { - var inprogressKeys = Object.keys(this.inprogress); + const inprogressKeys = Object.keys(this.inprogress); var upload; for (var i = 0; i < this.inprogress.length; ++i) { var k = inprogressKeys[i]; diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index 4a5dbab51e..6f46544bf4 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -24,6 +24,7 @@ import Modal from '../../../Modal'; import sdk from '../../../index'; import dis from '../../../dispatcher'; import {decryptFile} from '../../../utils/DecryptFile'; +import q from 'q'; module.exports = React.createClass({ displayName: 'MImageBody', @@ -36,6 +37,7 @@ module.exports = React.createClass({ getInitialState: function() { return { decryptedUrl: null, + decryptedThumbnailUrl: null, }; }, @@ -94,7 +96,9 @@ module.exports = React.createClass({ _getThumbUrl: function() { const content = this.props.mxEvent.getContent(); if (content.file !== undefined) { - // TODO: Decrypt and use the thumbnail file if one is present. + if (this.state.decryptedThumbnailUrl) { + return this.state.decryptedThumbnailUrl; + } return this.state.decryptedUrl; } else { return MatrixClientPeg.get().mxcUrlToHttp(content.url, 800, 600); @@ -106,15 +110,24 @@ module.exports = React.createClass({ this.fixupHeight(); const content = this.props.mxEvent.getContent(); if (content.file !== undefined && this.state.decryptedUrl === null) { - decryptFile(content.file).done((url) => { - this.setState({ - decryptedUrl: url, + var thumbnailPromise = q(null); + if (content.info.thumbnail_file) { + thumbnailPromise = decryptFile( + content.info.thumbnail_file + ); + } + thumbnailPromise.then((thumbnailUrl) => { + decryptFile(content.file).then((contentUrl) => { + this.setState({ + decryptedUrl: contentUrl, + decryptedThumbnailUrl: thumbnailUrl, + }); }); - }, (err) => { + }).catch((err) => { console.warn("Unable to decrypt attachment: ", err) // Set a placeholder image when we can't decrypt the image. this.refs.image.src = "img/warning.svg"; - }); + }).done(); } },