Matrix/element-web-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-web.git synced 2024-11-15 20:54:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Generate thumbnails when sending m.image and m.video messages. (#555)

Browse Source 
* Send a thumbnail when sending a m.image

* Use the 'thumbnail_file' when displaying encrypted images

* Whitespace

* Generate thumbnails for m.video

* Fix docstring, remove unused vars, use const

* Don't change the upload promise behaviour

* Polyfill for Canvas.toBlob to support older browsers

* Lowercase for integer types in jsdoc
This commit is contained in:
Mark Haines 2016-11-15 11:22:39 +00:00 committed by GitHub
parent 34df6ea242
commit 13f28e53e1
3 changed files with 219 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -42,6 +42,7 @@
},
"dependencies": {
"babel-runtime": "^6.11.6",
"blueimp-canvas-to-blob": "^3.5.0",
"browser-encrypt-attachment": "^0.1.0",
"browser-request": "^0.3.3",
"classnames": "^2.1.2",

258
src/ContentMessages.js
View File

@ -25,22 +25,87 @@ var Modal = require('./Modal');
var encrypt = require("browser-encrypt-attachment");
function infoForImageFile(imageFile) {
var deferred = q.defer();
// Polyfill for Canvas.toBlob API using Canvas.toDataURL
require("blueimp-canvas-to-blob");
const MAX_WIDTH = 800;
const MAX_HEIGHT = 600;
/**
* Create a thumbnail for a image DOM element.
* The image will be smaller than MAX_WIDTH and MAX_HEIGHT.
* The thumbnail will have the same aspect ratio as the original.
* Draws the element into a canvas using CanvasRenderingContext2D.drawImage
* Then calls Canvas.toBlob to get a blob object for the image data.
*
* Since it needs to calculate the dimensions of the source image and the
* thumbnailed image it returns an info object filled out with information
* about the original image and the thumbnail.
*
* @param {HTMLElement} element The element to thumbnail.
* @param {integer} inputWidth The width of the image in the input element.
* @param {integer} inputHeight the width of the image in the input element.
* @param {String} mimeType The mimeType to save the blob as.
* @return {Promise} A promise that resolves with an object with an info key
* and a thumbnail key.
*/
function createThumbnail(element, inputWidth, inputHeight, mimeType) {
const deferred = q.defer();
var targetWidth = inputWidth;
var targetHeight = inputHeight;
if (targetHeight > MAX_HEIGHT) {
targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight));
targetHeight = MAX_HEIGHT;
}
if (targetWidth > MAX_WIDTH) {
targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth));
targetWidth = MAX_WIDTH;
}
const canvas = document.createElement("canvas");
canvas.width = targetWidth;
canvas.height = targetHeight;
canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight);
canvas.toBlob(function(thumbnail) {
deferred.resolve({
info: {
thumbnail_info: {
w: targetWidth,
h: targetHeight,
mimetype: thumbnail.type,
size: thumbnail.size,
},
w: inputWidth,
h: inputHeight,
},
thumbnail: thumbnail
});
}, mimeType);
return deferred.promise;
}
/**
* Load a file into a newly created image element.
*
* @param {File} file The file to load in an image element.
* @return {Promise} A promise that resolves with the html image element.
*/
function loadImageElement(imageFile) {
const deferred = q.defer();
// Load the file into an html element
var img = document.createElement("img");
const img = document.createElement("img");
var reader = new FileReader();
const reader = new FileReader();
reader.onload = function(e) {
img.src = e.target.result;
// Once ready, returns its size
// Once ready, create a thumbnail
img.onload = function() {
deferred.resolve({
w: img.width,
h: img.height
});
deferred.resolve(img);
};
img.onerror = function(e) {
deferred.reject(e);
@ -54,22 +119,53 @@ function infoForImageFile(imageFile) {
return deferred.promise;
}
function infoForVideoFile(videoFile) {
var deferred = q.defer();
/**
* Read the metadata for an image file and create and upload a thumbnail of the image.
*
* @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
* @param {String} roomId The ID of the room the image will be uploaded in.
* @param {File} The image to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForImageFile(matrixClient, roomId, imageFile) {
var thumbnailType = "image/png";
if (imageFile.type == "image/jpeg") {
thumbnailType = "image/jpeg";
}
var imageInfo;
return loadImageElement(imageFile).then(function(img) {
return createThumbnail(img, img.width, img.height, thumbnailType);
}).then(function(result) {
imageInfo = result.info;
return uploadFile(matrixClient, roomId, result.thumbnail);
}).then(function(result) {
imageInfo.thumbnail_url = result.url;
imageInfo.thumbnail_file = result.file;
return imageInfo;
});
}
/**
* Load a file into a newly created video element.
*
* @param {File} file The file to load in an video element.
* @return {Promise} A promise that resolves with the video image element.
*/
function loadVideoElement(videoFile) {
const deferred = q.defer();
// Load the file into an html element
var video = document.createElement("video");
const video = document.createElement("video");
var reader = new FileReader();
const reader = new FileReader();
reader.onload = function(e) {
video.src = e.target.result;
// Once ready, returns its size
video.onloadedmetadata = function() {
deferred.resolve({
w: video.videoWidth,
h: video.videoHeight
});
// Wait until we have enough data to thumbnail the first frame.
video.onloadeddata = function() {
deferred.resolve(video);
};
video.onerror = function(e) {
deferred.reject(e);
@ -83,6 +179,30 @@ function infoForVideoFile(videoFile) {
return deferred.promise;
}
/**
* Read the metadata for a video file and create and upload a thumbnail of the video.
*
* @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
* @param {String} roomId The ID of the room the video will be uploaded to.
* @param {File} The video to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForVideoFile(matrixClient, roomId, videoFile) {
const thumbnailType = "image/jpeg";
var videoInfo;
return loadVideoElement(videoFile).then(function(video) {
return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
}).then(function(result) {
videoInfo = result.info;
return uploadFile(matrixClient, roomId, result.thumbnail);
}).then(function(result) {
videoInfo.thumbnail_url = result.url;
videoInfo.thumbnail_file = result.file;
return videoInfo;
});
}
/**
* Read the file as an ArrayBuffer.
* @return {Promise} A promise that resolves with an ArrayBuffer when the file
@ -101,6 +221,48 @@ function readFileAsArrayBuffer(file) {
return deferred.promise;
}
/**
* Upload the file to the content repository.
* If the room is encrypted then encrypt the file before uploading.
*
* @param {MatrixClient} matrixClient The matrix client to upload the file with.
* @param {String} roomId The ID of the room being uploaded to.
* @param {File} file The file to upload.
* @return {Promise} A promise that resolves with an object.
* If the file is unencrypted then the object will have a "url" key.
* If the file is encrypted then the object will have a "file" key.
*/
function uploadFile(matrixClient, roomId, file) {
if (matrixClient.isRoomEncrypted(roomId)) {
// If the room is encrypted then encrypt the file before uploading it.
// First read the file into memory.
return readFileAsArrayBuffer(file).then(function(data) {
// Then encrypt the file.
return encrypt.encryptAttachment(data);
}).then(function(encryptResult) {
// Record the information needed to decrypt the attachment.
const encryptInfo = encryptResult.info;
// Pass the encrypted data as a Blob to the uploader.
const blob = new Blob([encryptResult.data]);
return matrixClient.uploadContent(blob).then(function(url) {
// If the attachment is encrypted then bundle the URL along
// with the information needed to decrypt the attachment and
// add it under a file key.
encryptInfo.url = url;
if (file.type) {
encryptInfo.mimetype = file.type;
}
return {"file": encryptInfo};
});
});
} else {
return matrixClient.uploadContent(file).then(function(url) {
// If the attachment isn't encrypted then include the URL directly.
return {"url": url};
});
}
}
class ContentMessages {
constructor() {
@ -109,7 +271,7 @@ class ContentMessages {
}
sendContentToRoom(file, roomId, matrixClient) {
var content = {
const content = {
body: file.name,
info: {
size: file.size,
@ -121,13 +283,14 @@ class ContentMessages {
content.info.mimetype = file.type;
}
var def = q.defer();
const def = q.defer();
if (file.type.indexOf('image/') == 0) {
content.msgtype = 'm.image';
infoForImageFile(file).then(imageInfo=>{
infoForImageFile(matrixClient, roomId, file).then(imageInfo=>{
extend(content.info, imageInfo);
def.resolve();
}, error=>{
console.error(error);
content.msgtype = 'm.file';
def.resolve();
});
@ -136,7 +299,7 @@ class ContentMessages {
def.resolve();
} else if (file.type.indexOf('video/') == 0) {
content.msgtype = 'm.video';
infoForVideoFile(file).then(videoInfo=>{
infoForVideoFile(matrixClient, roomId, file).then(videoInfo=>{
extend(content.info, videoInfo);
def.resolve();
}, error=>{
@ -148,35 +311,23 @@ class ContentMessages {
def.resolve();
}
var upload = {
const upload = {
fileName: file.name,
roomId: roomId,
total: 0,
loaded: 0
loaded: 0,
};
this.inprogress.push(upload);
dis.dispatch({action: 'upload_started'});
var encryptInfo = null;
var error;
var self = this;
return def.promise.then(function() {
if (matrixClient.isRoomEncrypted(roomId)) {
// If the room is encrypted then encrypt the file before uploading it.
// First read the file into memory.
upload.promise = readFileAsArrayBuffer(file).then(function(data) {
// Then encrypt the file.
return encrypt.encryptAttachment(data);
}).then(function(encryptResult) {
// Record the information needed to decrypt the attachment.
encryptInfo = encryptResult.info;
// Pass the encrypted data as a Blob to the uploader.
var blob = new Blob([encryptResult.data]);
return matrixClient.uploadContent(blob);
upload.promise = uploadFile(
matrixClient, roomId, file
).then(function(result) {
content.file = result.file;
content.url = result.url;
});
} else {
upload.promise = matrixClient.uploadContent(file);
}
return upload.promise;
}).progress(function(ev) {
if (ev) {
@ -185,19 +336,6 @@ class ContentMessages {
dis.dispatch({action: 'upload_progress', upload: upload});
}
}).then(function(url) {
if (encryptInfo === null) {
// If the attachment isn't encrypted then include the URL directly.
content.url = url;
} else {
// If the attachment is encrypted then bundle the URL along
// with the information needed to decrypt the attachment and
// add it under a file key.
encryptInfo.url = url;
if (file.type) {
encryptInfo.mimetype = file.type;
}
content.file = encryptInfo;
}
return matrixClient.sendMessage(roomId, content);
}, function(err) {
error = err;
@ -212,12 +350,12 @@ class ContentMessages {
description: desc
});
}
}).finally(function() {
var inprogressKeys = Object.keys(self.inprogress);
for (var i = 0; i < self.inprogress.length; ++i) {
}).finally(() => {
const inprogressKeys = Object.keys(this.inprogress);
for (var i = 0; i < this.inprogress.length; ++i) {
var k = inprogressKeys[i];
if (self.inprogress[k].promise === upload.promise) {
self.inprogress.splice(k, 1);
if (this.inprogress[k].promise === upload.promise) {
this.inprogress.splice(k, 1);
break;
}
}
@ -235,7 +373,7 @@ class ContentMessages {
}
cancelUpload(promise) {
var inprogressKeys = Object.keys(this.inprogress);
const inprogressKeys = Object.keys(this.inprogress);
var upload;
for (var i = 0; i < this.inprogress.length; ++i) {
var k = inprogressKeys[i];

23
src/components/views/messages/MImageBody.js
View File

@ -24,6 +24,7 @@ import Modal from '../../../Modal';
import sdk from '../../../index';
import dis from '../../../dispatcher';
import {decryptFile} from '../../../utils/DecryptFile';
import q from 'q';
module.exports = React.createClass({
displayName: 'MImageBody',
@ -36,6 +37,7 @@ module.exports = React.createClass({
getInitialState: function() {
return {
decryptedUrl: null,
decryptedThumbnailUrl: null,
};
},
@ -94,7 +96,9 @@ module.exports = React.createClass({
_getThumbUrl: function() {
const content = this.props.mxEvent.getContent();
if (content.file !== undefined) {
// TODO: Decrypt and use the thumbnail file if one is present.
if (this.state.decryptedThumbnailUrl) {
return this.state.decryptedThumbnailUrl;
}
return this.state.decryptedUrl;
} else {
return MatrixClientPeg.get().mxcUrlToHttp(content.url, 800, 600);
@ -106,15 +110,24 @@ module.exports = React.createClass({
this.fixupHeight();
const content = this.props.mxEvent.getContent();
if (content.file !== undefined && this.state.decryptedUrl === null) {
decryptFile(content.file).done((url) => {
var thumbnailPromise = q(null);
if (content.info.thumbnail_file) {
thumbnailPromise = decryptFile(
content.info.thumbnail_file
);
}
thumbnailPromise.then((thumbnailUrl) => {
decryptFile(content.file).then((contentUrl) => {
this.setState({
decryptedUrl: url,
decryptedUrl: contentUrl,
decryptedThumbnailUrl: thumbnailUrl,
});
}, (err) => {
});
}).catch((err) => {
console.warn("Unable to decrypt attachment: ", err)
// Set a placeholder image when we can't decrypt the image.
this.refs.image.src = "img/warning.svg";
});
}).done();
}
},