CryptoStore migration: step5: convert serialized classes to other classes

legacy MXDeviceInfo and MXOlmInboundGroupSession2

matrix-sdk-android/src/debug/java/im/vector/matrix/android/internal/database/RealmDebugTools.kt

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.database
+
+import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.CryptoMetadataEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.CryptoRoomEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.DeviceInfoEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.GossipingEventEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.IncomingGossipingRequestEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.KeyInfoEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.KeysBackupDataEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.OlmSessionEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.OutgoingGossipingRequestEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.TrustLevelEntity
+import im.vector.matrix.android.internal.crypto.store.db.model.UserEntity
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.kotlin.where
+import timber.log.Timber
+
+object RealmDebugTools {
+    /**
+     * Log info about the crypto DB
+     */
+    fun dumpCryptoDb(realmConfiguration: RealmConfiguration) {
+        Realm.getInstance(realmConfiguration).use {
+            Timber.d("Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}")
+
+            val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) }
+            Timber.d("Realm encryption key : $key")
+
+            // Check if we have data
+            Timber.e("Realm is empty: ${it.isEmpty}")
+
+            Timber.d("Realm has CryptoMetadataEntity: ${it.where<CryptoMetadataEntity>().count()}")
+            Timber.d("Realm has CryptoRoomEntity: ${it.where<CryptoRoomEntity>().count()}")
+            Timber.d("Realm has DeviceInfoEntity: ${it.where<DeviceInfoEntity>().count()}")
+            Timber.d("Realm has KeysBackupDataEntity: ${it.where<KeysBackupDataEntity>().count()}")
+            Timber.d("Realm has OlmInboundGroupSessionEntity: ${it.where<OlmInboundGroupSessionEntity>().count()}")
+            Timber.d("Realm has OlmSessionEntity: ${it.where<OlmSessionEntity>().count()}")
+            Timber.d("Realm has UserEntity: ${it.where<UserEntity>().count()}")
+            Timber.d("Realm has KeyInfoEntity: ${it.where<KeyInfoEntity>().count()}")
+            Timber.d("Realm has CrossSigningInfoEntity: ${it.where<CrossSigningInfoEntity>().count()}")
+            Timber.d("Realm has TrustLevelEntity: ${it.where<TrustLevelEntity>().count()}")
+            Timber.d("Realm has GossipingEventEntity: ${it.where<GossipingEventEntity>().count()}")
+            Timber.d("Realm has IncomingGossipingRequestEntity: ${it.where<IncomingGossipingRequestEntity>().count()}")
+            Timber.d("Realm has OutgoingGossipingRequestEntity: ${it.where<OutgoingGossipingRequestEntity>().count()}")
+            Timber.d("Realm has MyDeviceLastSeenInfoEntity: ${it.where<MyDeviceLastSeenInfoEntity>().count()}")
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -41,8 +41,10 @@ import im.vector.matrix.android.internal.crypto.store.db.model.UserEntityFields
 import im.vector.matrix.android.internal.di.SerializeNulls
 import io.realm.DynamicRealm
 import io.realm.RealmMigration
+import org.matrix.androidsdk.crypto.data.MXOlmInboundGroupSession2
 import timber.log.Timber
 import javax.inject.Inject
+import org.matrix.androidsdk.crypto.data.MXDeviceInfo as LegacyMXDeviceInfo
 
 internal class RealmCryptoStoreMigration @Inject constructor(private val crossSigningKeysMapper: CrossSigningKeysMapper) : RealmMigration {
 
@@ -146,6 +148,52 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
 
         realm.schema.get("CryptoRoomEntity")
                 ?.addField(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, Boolean::class.java)
+                ?.setRequired(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, false)
+
+        // Convert format of MXDeviceInfo, package has to be the same.
+        realm.schema.get("DeviceInfoEntity")
+                ?.transform { obj ->
+                    try {
+                        val oldSerializedData = obj.getString("deviceInfoData")
+                        deserializeFromRealm<LegacyMXDeviceInfo>(oldSerializedData)?.let { legacyMxDeviceInfo ->
+                            val newMxDeviceInfo = MXDeviceInfo(
+                                    deviceId = legacyMxDeviceInfo.deviceId,
+                                    userId = legacyMxDeviceInfo.userId,
+                                    algorithms = legacyMxDeviceInfo.algorithms,
+                                    keys = legacyMxDeviceInfo.keys,
+                                    signatures = legacyMxDeviceInfo.signatures,
+                                    unsigned = legacyMxDeviceInfo.unsigned,
+                                    verified = legacyMxDeviceInfo.mVerified
+                            )
+
+                            obj.setString("deviceInfoData", serializeForRealm(newMxDeviceInfo))
+                        }
+                    } catch (e: Exception) {
+                        Timber.e(e, "Error")
+                    }
+                }
+
+        // Convert MXOlmInboundGroupSession2 to OlmInboundGroupSessionWrapper2
+        realm.schema.get("OlmInboundGroupSessionEntity")
+                ?.transform { obj ->
+                    try {
+                        val oldSerializedData = obj.getString("olmInboundGroupSessionData")
+                        deserializeFromRealm<MXOlmInboundGroupSession2>(oldSerializedData)?.let { mxOlmInboundGroupSession2 ->
+                            val newOlmInboundGroupSessionWrapper2 = OlmInboundGroupSessionWrapper2()
+                                    .apply {
+                                        olmInboundGroupSession = mxOlmInboundGroupSession2.mSession
+                                        roomId = mxOlmInboundGroupSession2.mRoomId
+                                        senderKey = mxOlmInboundGroupSession2.mSenderKey
+                                        keysClaimed = mxOlmInboundGroupSession2.mKeysClaimed
+                                        forwardingCurve25519KeyChain = mxOlmInboundGroupSession2.mForwardingCurve25519KeyChain
+                                    }
+
+                            obj.setString("olmInboundGroupSessionData", serializeForRealm(newOlmInboundGroupSessionWrapper2))
+                        }
+                    } catch (e: Exception) {
+                        Timber.e(e, "Error")
+                    }
+                }
     }
 
     // Version 4L added Cross Signing info persistence
@@ -364,7 +412,7 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                     deviceList.addAll(distinct)
                 }
             } catch (failure: Throwable) {
-                Timber.w(failure, "Crypto Data base migration error for migrateTo6")
+                Timber.w(failure, "Crypto Data base migration error for migrateTo9")
             }
         }
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/legacy/DefaultLegacySessionImporter.kt

@@ -22,7 +22,6 @@ import im.vector.matrix.android.api.auth.data.DiscoveryInformation
 import im.vector.matrix.android.api.auth.data.HomeServerConnectionConfig
 import im.vector.matrix.android.api.auth.data.SessionParams
 import im.vector.matrix.android.api.auth.data.WellKnownBaseConfig
-import im.vector.matrix.android.api.extensions.tryThis
 import im.vector.matrix.android.api.legacy.LegacySessionImporter
 import im.vector.matrix.android.internal.auth.SessionParamsStore
 import im.vector.matrix.android.internal.crypto.store.db.RealmCryptoStoreMigration
@@ -49,6 +48,11 @@ internal class DefaultLegacySessionImporter @Inject constructor(
 
     private val loginStorage = LoginStorage(context)
 
+    companion object {
+        // During development, set to false to play several times the migration
+        private var DELETE_PREVIOUS_DATA = true
+    }
+
     override fun process() {
         Timber.d("Migration: Importing legacy session")
 
@@ -64,7 +68,7 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                 importCredentials(legacyConfig)
             } catch (t: Throwable) {
                 // It can happen in case of partial migration. To test, do not return
-                Timber.e(t, "Error importing credential")
+                Timber.e(t, "Migration: Error importing credential")
             }
 
             Timber.d("Migration: importing crypto DB")
@@ -72,23 +76,25 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                 importCryptoDb(legacyConfig)
             } catch (t: Throwable) {
                 // It can happen in case of partial migration. To test, do not return
-                Timber.e(t, "Error importing crypto DB")
+                Timber.e(t, "Migration: Error importing crypto DB")
             }
 
-            Timber.d("Migration: clear file system")
+            if (DELETE_PREVIOUS_DATA) {
-            try {
+                try {
-                clearFileSystem(legacyConfig)
+                    Timber.d("Migration: clear file system")
-            } catch (t: Throwable) {
+                    clearFileSystem(legacyConfig)
-                // It can happen in case of partial migration. To test, do not return
+                } catch (t: Throwable) {
-                Timber.e(t, "Error clearing filesystem")
+                    Timber.e(t, "Migration: Error clearing filesystem")
-            }
+                }
-
+                try {
-            Timber.d("Migration: clear shared prefs")
+                    Timber.d("Migration: clear shared prefs")
-            try {
+                    clearSharedPrefs()
-                clearSharedPrefs()
+                } catch (t: Throwable) {
-            } catch (t: Throwable) {
+                    Timber.e(t, "Migration: Error clearing shared prefs")
-                // It can happen in case of partial migration. To test, do not return
+                }
-                Timber.e(t, "Error clearing filesystem")
+            } else {
+                Timber.d("Migration: clear file system - DEACTIVATED")
+                Timber.d("Migration: clear shared prefs - DEACTIVATED")
             }
         }
     }
@@ -104,8 +110,7 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                         deviceId = legacyConfig.credentials.deviceId,
                         discoveryInformation = legacyConfig.credentials.wellKnown?.let { wellKnown ->
                             // Note credentials.wellKnown is not serialized in the LoginStorage, so this code is a bit useless...
-                            if (wellKnown.homeServer?.baseURL != null
+                            if (wellKnown.homeServer?.baseURL != null || wellKnown.identityServer?.baseURL != null) {
-                                    || wellKnown.identityServer?.baseURL != null) {
                                 DiscoveryInformation(
                                         homeServer = wellKnown.homeServer?.baseURL?.let { WellKnownBaseConfig(baseURL = it) },
                                         identityServer = wellKnown.identityServer?.baseURL?.let { WellKnownBaseConfig(baseURL = it) }
@@ -157,7 +162,6 @@ internal class DefaultLegacySessionImporter @Inject constructor(
         newLocation.deleteRecursively()
         newLocation.mkdirs()
 
-        // TODO Check if file exists first?
         Timber.d("Migration: create legacy realm configuration")
 
         val realmConfiguration = RealmConfiguration.Builder()
@@ -166,7 +170,6 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                 .modules(RealmCryptoStoreModule())
                 .schemaVersion(RealmCryptoStoreMigration.CRYPTO_STORE_SCHEMA_VERSION)
                 .migration(realmCryptoStoreMigration)
-                // .initialData(CryptoFileStoreImporter(enableFileEncryption, context, credentials))
                 .build()
 
         Timber.d("Migration: copy DB to encrypted DB")
@@ -185,7 +188,7 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                 File(context.filesDir, "MXFileStore"),
                 // Previous (and very old) file crypto store
                 File(context.filesDir, "MXFileCryptoStore"),
-                // Draft. They will be lost, this is sad TODO handle them?
+                // Draft. They will be lost, this is sad but we assume it
                 File(context.filesDir, "MXLatestMessagesStore"),
                 // Media storage
                 File(context.filesDir, "MXMediaStore"),
@@ -196,7 +199,11 @@ internal class DefaultLegacySessionImporter @Inject constructor(
                 // Crypto store
                 File(context.filesDir, cryptoFolder)
         ).forEach { file ->
-            tryThis { file.deleteRecursively() }
+            try {
+                file.deleteRecursively()
+            } catch (t: Throwable) {
+                Timber.e(t, "Migration: unable to delete $file")
+            }
         }
     }
 

matrix-sdk-android/src/main/java/org/matrix/androidsdk/crypto/data/MXDeviceInfo.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.androidsdk.crypto.data;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/*
+ * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
+ */
+
+public class MXDeviceInfo implements Serializable {
+    private static final long serialVersionUID = 20129670646382964L;
+
+    // This device is a new device and the user was not warned it has been added.
+    public static final int DEVICE_VERIFICATION_UNKNOWN = -1;
+
+    // The user has not yet verified this device.
+    public static final int DEVICE_VERIFICATION_UNVERIFIED = 0;
+
+    // The user has verified this device.
+    public static final int DEVICE_VERIFICATION_VERIFIED = 1;
+
+    // The user has blocked this device.
+    public static final int DEVICE_VERIFICATION_BLOCKED = 2;
+
+    /**
+     * The id of this device.
+     */
+    public String deviceId;
+
+    /**
+     * the user id
+     */
+    public String userId;
+
+    /**
+     * The list of algorithms supported by this device.
+     */
+    public List<String> algorithms;
+
+    /**
+     * A map from <key type>:<id> to <base64-encoded key>>.
+     */
+    public Map<String, String> keys;
+
+    /**
+     * The signature of this MXDeviceInfo.
+     * A map from <key type>:<device_id> to <base64-encoded key>>.
+     */
+    public Map<String, Map<String, String>> signatures;
+
+    /*
+     * Additional data from the home server.
+     */
+    public Map<String, Object> unsigned;
+
+    /**
+     * Verification state of this device.
+     */
+    public int mVerified;
+
+    /**
+     * Constructor
+     */
+    public MXDeviceInfo() {
+        mVerified = DEVICE_VERIFICATION_UNKNOWN;
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/androidsdk/crypto/data/MXOlmInboundGroupSession2.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.androidsdk.crypto.data;
+
+import org.matrix.olm.OlmInboundGroupSession;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/*
+ * IMPORTANT: This class is imported from Riot-Android to be able to perform a migration. Do not use it for any other purpose
+ */
+
+/**
+ * This class adds more context to a OLMInboundGroupSession object.
+ * This allows additional checks. The class implements NSCoding so that the context can be stored.
+ */
+public class MXOlmInboundGroupSession2 implements Serializable {
+    // define a serialVersionUID to avoid having to redefine the class after updates
+    private static final long serialVersionUID = 201702011617L;
+
+    // The associated olm inbound group session.
+    public OlmInboundGroupSession mSession;
+
+    // The room in which this session is used.
+    public String mRoomId;
+
+    // The base64-encoded curve25519 key of the sender.
+    public String mSenderKey;
+
+    // Other keys the sender claims.
+    public Map<String, String> mKeysClaimed;
+
+    // Devices which forwarded this session to us (normally empty).
+    public List<String> mForwardingCurve25519KeyChain = new ArrayList<>();
+}