Add tool to migrate logins between origins

App checks at startup for an existing session, if there isn't one,
it will start the tool to check for a login in the file:// origin.
If there is one, it will copy the login over to the vector://vector
origin.

In principle this could also be used to migrate logins between
other origins on the web if this were ever required.

This includes a minified copy of the browserified js-sdk with
a getAllEndToEndSessions() function added to the crypto store
(https://github.com/matrix-org/matrix-js-sdk/pull/812). This is
not great, but for a short-lived tool this seems better than
introducing more entry points into webpack only used for the
electron app.

electron_app/src/electron-main.js

@@ -32,6 +32,7 @@ const tray = require('./tray');
 const vectorMenu = require('./vectormenu');
 const webContentsHandler = require('./webcontents-handler');
 const updater = require('./updater');
+const { migrateFromOldOrigin } = require('./originMigrator');
 
 const windowStateKeeper = require('electron-window-state');
 
@@ -110,7 +111,7 @@ autoUpdater.on('update-downloaded', (ev, releaseNotes, releaseName, releaseDate,
     });
 });
 
-ipcMain.on('ipcCall', function(ev, payload) {
+ipcMain.on('ipcCall', async function(ev, payload) {
     if (!mainWindow) return;
 
     const args = payload.args || [];
@@ -141,10 +142,13 @@ ipcMain.on('ipcCall', function(ev, payload) {
             } else {
                 mainWindow.focus();
             }
+        case 'origin_migrate':
+            await migrateFromOldOrigin();
+            break;
         default:
             mainWindow.webContents.send('ipcReply', {
                 id: payload.id,
-                error: new Error("Unknown IPC Call: "+payload.name),
+                error: "Unknown IPC Call: " + payload.name,
             });
             return;
     }
@@ -210,19 +214,42 @@ app.on('ready', () => {
         }
 
         const target = parsedUrl.pathname.split('/');
+
+        // path starts with a '/'
+        if (target[0] !== '') {
+            callback({error: -6}); // FILE_NOT_FOUND
+            return;
+        }
+
         if (target[target.length - 1] == '') {
             target[target.length - 1] = 'index.html';
         }
 
+        let baseDir;
+        // first part of the path determines where we serve from
+        if (target[1] === 'origin_migrator_dest') {
+            // the origin migrator destination page
+            // (only the destination script needs to come from the
+            // custom protocol: the source part is loaded from a
+            // file:// as that's the origin we're migrating from).
+            baseDir = __dirname + "/../../origin_migrator/dest";
+        } else if (target[1] === 'webapp') {
+            baseDir = __dirname + "/../../webapp";
+        } else {
+            callback({error: -6}); // FILE_NOT_FOUND
+            return;
+        }
+
         // Normalise the base dir and the target path separately, then make sure
         // the target path isn't trying to back out beyond its root
-        const appBaseDir = path.normalize(__dirname + "/../../webapp");
-        const relTarget = path.normalize(path.join(...target));
+        baseDir = path.normalize(baseDir);
+
+        const relTarget = path.normalize(path.join(...target.slice(2)));
         if (relTarget.startsWith('..')) {
             callback({error: -6}); // FILE_NOT_FOUND
             return;
         }
-        const absTarget = path.join(appBaseDir, relTarget);
+        const absTarget = path.join(baseDir, relTarget);
 
         callback({
             path: absTarget,
@@ -231,8 +258,6 @@ app.on('ready', () => {
         if (error) console.error('Failed to register protocol')
     });
 
-
-
     if (vectorConfig['update_base_url']) {
         console.log(`Starting auto update with base URL: ${vectorConfig['update_base_url']}`);
         updater.start(vectorConfig['update_base_url']);
@@ -271,7 +296,7 @@ app.on('ready', () => {
             webgl: false,
         },
     });
-    mainWindow.loadURL('vector://vector/');
+    mainWindow.loadURL('vector://vector/webapp/');
     Menu.setApplicationMenu(vectorMenu);
 
     // explicitly hide because setApplicationMenu on Linux otherwise shows...

electron_app/src/originMigrator.js

@@ -0,0 +1,62 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const { BrowserWindow, ipcMain } = require('electron');
+const path = require('path');
+
+async function migrateFromOldOrigin() {
+    console.log("Attempting to migrate data between origins");
+
+    // We can use the same preload script: we just need ipcRenderer exposed
+    const preloadScript = path.normalize(`${__dirname}/preload.js`);
+    await new Promise(resolve => {
+        const migrateWindow = new BrowserWindow({
+            show: false,
+            webPreferences: {
+                preload: preloadScript,
+                nodeIntegration: false,
+                sandbox: true,
+                enableRemoteModule: false,
+                webgl: false,
+            },
+        });
+        ipcMain.on('origin_migration_complete', (e, success, sentSummary, storedSummary) => {
+            if (success) {
+                console.log("Origin migration completed successfully!");
+            } else {
+                console.error("Origin migration failed!");
+            }
+            console.error("Data sent", sentSummary);
+            console.error("Data stored", storedSummary);
+            migrateWindow.close();
+            resolve();
+        });
+        ipcMain.on('origin_migration_nodata', (e) => {
+            console.log("No session to migrate from old origin");
+            migrateWindow.close();
+            resolve();
+        });
+        // Normalise the path because in the distribution, __dirname will be inside the
+        // electron asar.
+        const sourcePagePath = path.normalize(__dirname + '/../../origin_migrator/source.html');
+        console.log("Loading path: " + sourcePagePath);
+        migrateWindow.loadURL('file://' + sourcePagePath);
+    });
+}
+
+module.exports = {
+    migrateFromOldOrigin,
+};

origin_migrator/dest/dest.html

@@ -0,0 +1,6 @@
+<html>
+<body>
+<script src="browser-matrix.min.js"></script>
+<script src="dest.js"></script>
+</body>
+</html>

origin_migrator/dest/dest.js

@@ -0,0 +1,125 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const SOURCE_ORIGIN = 'file://';
+
+const IndexedDBCryptoStore = window.matrixcs.IndexedDBCryptoStore;
+const cryptoStore = new IndexedDBCryptoStore(window.indexedDB, 'matrix-js-sdk:crypto');
+
+let accountStored = 0;
+let sessionsStored = 0;
+let inboundGroupSessionsStored = 0;
+let deviceDataStored = 0;
+let roomsStored = 0;
+let localStorageKeysStored = 0;
+
+const promises = [];
+
+async function onMessage(e) {
+    if (e.origin !== SOURCE_ORIGIN) return;
+
+    const data = e.data.data; // bleh, naming clash
+    switch (e.data.cmd) {
+        case 'init':
+            // start with clean stores before we migrate data in
+            window.localStorage.clear();
+            await cryptoStore.deleteAllData();
+
+            e.source.postMessage({
+                cmd: 'initOK',
+            }, SOURCE_ORIGIN);
+            break;
+        case 'storeAccount':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    cryptoStore.storeAccount(txn, data);
+                },
+            ).then(() => {
+                ++accountStored;
+            }));
+            break;
+        case 'storeSessions':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
+                (txn) => {
+                    for (const sess of data) {
+                        cryptoStore.storeEndToEndSession(sess.deviceKey, sess.sessionId, sess, txn);
+                    }
+                },
+            ).then(() => {
+                sessionsStored += data.length;
+            }));
+            break;
+        case 'storeInboundGroupSessions':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                (txn) => {
+                    for (const sess of data) {
+                        cryptoStore.addEndToEndInboundGroupSession(
+                            sess.senderKey, sess.sessionId, sess.sessionData, txn,
+                        );
+                    }
+                },
+            ).then(() => {
+                inboundGroupSessionsStored += data.length;
+            }));
+            break;
+        case 'storeDeviceData':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA],
+                (txn) => {
+                    cryptoStore.storeEndToEndDeviceData(data, txn);
+                },
+            ).then(() => {
+                ++deviceDataStored;
+            }));
+            break;
+        case 'storeRooms':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ROOMS],
+                (txn) => {
+                    for (const [roomId, roomInfo] of Object.entries(data)) {
+                        cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+                    }
+                },
+            ).then(() => {
+                ++roomsStored;
+            }));
+            break;
+        case 'storeLocalStorage':
+            window.localStorage.setItem(data.key, data.val);
+            ++localStorageKeysStored;
+            break;
+        case 'getSummary':
+            await Promise.all(promises);
+            e.source.postMessage({
+                cmd: 'summary',
+                data: {
+                    accountStored,
+                    sessionsStored,
+                    inboundGroupSessionsStored,
+                    deviceDataStored,
+                    roomsStored,
+                    localStorageKeysStored,
+                },
+            }, SOURCE_ORIGIN);
+            break;
+    }
+}
+
+window.addEventListener('message', onMessage);
+

origin_migrator/source.html

@@ -0,0 +1,7 @@
+<html>
+<body>
+<script src="dest/browser-matrix.min.js"></script>
+<script src="source.js"></script>
+<iframe name="dest" src="vector://vector/origin_migrator_dest/dest.html" onload="doMigrate()"></iframe>
+</body>
+</html>

origin_migrator/source.js

@@ -0,0 +1,210 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const TARGET_ORIGIN = 'vector://vector';
+const BATCH_SIZE = 500;
+let destFrame;
+
+let initResolver = null;
+let getSummaryResolver = null;
+
+function onMessage(e) {
+    if (e.origin !== TARGET_ORIGIN) return;
+
+    if (e.data.cmd === 'initOK' && initResolver) {
+        initResolver();
+        initResolver = null;
+    } else if (e.data.cmd === 'summary' && getSummaryResolver) {
+        getSummaryResolver(e.data.data);
+        getSummaryResolver = null;
+    }
+}
+
+async function initDestFrame() {
+    return new Promise(resolve => {
+        initResolver = resolve;
+        destFrame.postMessage({
+            cmd: 'init', 
+        }, TARGET_ORIGIN);
+    });
+}
+
+async function getSummary() {
+    return new Promise(resolve => {
+        getSummaryResolver = resolve;
+        destFrame.postMessage({
+            cmd: 'getSummary', 
+        }, TARGET_ORIGIN);
+    });
+}
+
+async function doMigrate() {
+    let accountSent = 0;
+    let sessionsSent = 0;
+    let inboundGroupSessionsSent = 0;
+    let deviceDataSent = 0;
+    let roomsSent = 0;
+    let localStorageKeysSent = 0;
+
+    if (!window.ipcRenderer) {
+        console.error("ipcRenderer not found");
+        return;
+    }
+
+    if (window.localStorage.getItem('mx_user_id') === null) {
+        window.ipcRenderer.send("origin_migration_nodata");
+        return;
+    }
+
+    destFrame = window.parent.frames.dest;
+
+    await initDestFrame();
+
+    const IndexedDBCryptoStore = window.matrixcs.IndexedDBCryptoStore;
+
+    const cryptoStore = new IndexedDBCryptoStore(window.indexedDB, 'matrix-js-sdk:crypto');
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            cryptoStore.getAccount(txn, (account) => {
+                destFrame.postMessage({
+                    cmd: 'storeAccount',
+                    data: account,
+                }, TARGET_ORIGIN);
+                ++accountSent;
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_SESSIONS],
+        (txn) => {
+            let sessBatch = [];
+            cryptoStore.getAllEndToEndSessions(txn, (sessInfo) => {
+                if (sessInfo) {
+                    ++sessionsSent;
+                    sessBatch.push(sessInfo);
+                }
+                if (sessBatch.length >= BATCH_SIZE || sessInfo === null) {
+                    destFrame.postMessage({
+                        cmd: 'storeSessions',
+                        data: sessBatch,
+                    }, TARGET_ORIGIN);
+                    sessBatch = [];
+                }
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+        (txn) => {
+            let sessBatch = [];
+            cryptoStore.getAllEndToEndInboundGroupSessions(txn, (sessInfo) => {
+                if (sessInfo) {
+                    ++inboundGroupSessionsSent;
+                    sessBatch.push(sessInfo);
+                }
+                if (sessBatch.length >= BATCH_SIZE || sessInfo === null) {
+                    destFrame.postMessage({
+                        cmd: 'storeInboundGroupSessions',
+                        data: sessBatch,
+                    }, TARGET_ORIGIN);
+                    sessBatch = [];
+                }
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA],
+        (txn) => {
+            cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                destFrame.postMessage({
+                    cmd: 'storeDeviceData',
+                    data: deviceData,
+                }, TARGET_ORIGIN);
+                ++deviceDataSent;
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ROOMS],
+        (txn) => {
+            cryptoStore.getEndToEndRooms(txn, (rooms) => {
+                destFrame.postMessage({
+                    cmd: 'storeRooms',
+                    data: rooms,
+                }, TARGET_ORIGIN);
+                ++roomsSent;
+            });
+        },
+    );
+
+    // we don't bother migrating;
+    // * sync data (we can just initialsync again)
+    // * logs
+    // * key requests (worst case they'll just be re-sent)
+    // * sessions needing backup (feature isn't available on Electron)
+
+    for (let i = 0; i < window.localStorage.length; ++i) {
+        const key = window.localStorage.key(i);
+        const val = window.localStorage.getItem(key);
+        
+        destFrame.postMessage({
+            cmd: 'storeLocalStorage',
+            data: { key, val },
+        }, TARGET_ORIGIN);
+        ++localStorageKeysSent;
+    }
+
+    const summary = await getSummary();
+    let success = false;
+    if (
+        summary.accountStored === accountSent &&
+        summary.sessionsStored === sessionsSent &&
+        summary.inboundGroupSessionsStored === inboundGroupSessionsSent &&
+        summary.deviceDataStored === deviceDataSent &&
+        summary.roomsStored === roomsSent &&
+        summary.localStorageKeysStored === localStorageKeysSent
+    ) {
+        success = true;
+        window.localStorage.clear();
+        await cryptoStore.deleteAllData();
+
+        // we don't bother migrating them, but also blow away the sync & logs db,
+        // otherwise they'll just hang about taking up space
+        await new Promise(resolve => {
+            const req = window.indexedDB.deleteDatabase('matrix-js-sdk:riot-web-sync');
+            req.onsuccess = resolve;
+            req.onerror = resolve;
+        });
+        await new Promise(resolve => {
+            const req = window.indexedDB.deleteDatabase('logs');
+            req.onsuccess = resolve;
+            req.onerror = resolve;
+        });
+    }
+
+    window.ipcRenderer.send("origin_migration_complete", success, {
+        accountSent, sessionsSent, inboundGroupSessionsSent,
+        deviceDataSent, roomsSent, localStorageKeysSent,
+    }, summary);
+}
+
+window.addEventListener('message', onMessage);

package.json

@@ -157,7 +157,8 @@
       "img/**"
     ],
     "extraResources": [
-      "webapp/**/*"
+      "webapp/**/*",
+      "origin_migrator/**/*"
     ],
     "linux": {
       "target": "deb",

src/vector/index.js

@@ -224,7 +224,16 @@ async function loadApp() {
     // set the platform for react sdk
     if (window.ipcRenderer) {
         console.log("Using Electron platform");
-        PlatformPeg.set(new ElectronPlatform());
+        const plaf = new ElectronPlatform();
+        PlatformPeg.set(plaf);
+
+        // Electron only: see if we need to do a one-time data
+        // migration
+        if (window.localStorage.getItem('mx_user_id') === null) {
+            console.log("Migrating session from old origin...");
+            await plaf.migrateFromOldOrigin();
+            console.log("Origin migration complete");
+        }
     } else {
         console.log("Using Web platform");
         PlatformPeg.set(new WebPlatform());

src/vector/platform/ElectronPlatform.js

@@ -224,6 +224,10 @@ export default class ElectronPlatform extends VectorBasePlatform {
         window.location.reload(false);
     }
 
+    async migrateFromOldOrigin() {
+        return this._ipcCall('origin_migrate');
+    }
+
     async _ipcCall(name, ...args) {
         const ipcCallId = ++this._nextIpcCallId;
         return new Promise((resolve, reject) => {

src/vector/platform/VectorBasePlatform.js

@@ -149,4 +149,12 @@ export default class VectorBasePlatform extends BasePlatform {
     getDefaultDeviceDisplayName(): string {
         return _t("Unknown device");
     }
+
+    /**
+     * Migrate account data from a previous origin
+     * Used only for the electron app
+     */
+    async migrateFromOldOrigin() {
+        return false;
+    }
 }