From cf3dbb378e72264c248d73a96210516b04f54ce5 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 26 May 2020 22:06:51 +0200 Subject: [PATCH 1/2] Get uploaded files for e2e rooms, from local DB --- .../database/query/TimelineEventFilter.kt | 7 +++ .../room/uploads/DefaultUploadsService.kt | 6 +- .../session/room/uploads/GetUploadsTask.kt | 58 +++++++++++++++---- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventFilter.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventFilter.kt index ea8122bc6d..14560ead85 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventFilter.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventFilter.kt @@ -28,6 +28,13 @@ internal object TimelineEventFilter { internal const val RESPONSE = """{*"m.relates_to"*"rel_type":*"m.response"*}""" } + /** + * To apply to Event.decryptionResultJson + */ + internal object DecryptedContent { + internal const val URL = """{*"file":*"url":*}""" + } + /** * To apply to Event.unsigned */ diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/DefaultUploadsService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/DefaultUploadsService.kt index dd8269a079..6ec9abf8e2 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/DefaultUploadsService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/DefaultUploadsService.kt @@ -19,6 +19,7 @@ package im.vector.matrix.android.internal.session.room.uploads import com.squareup.inject.assisted.Assisted import com.squareup.inject.assisted.AssistedInject import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.session.crypto.CryptoService import im.vector.matrix.android.api.session.room.uploads.GetUploadsResult import im.vector.matrix.android.api.session.room.uploads.UploadsService import im.vector.matrix.android.api.util.Cancelable @@ -28,7 +29,8 @@ import im.vector.matrix.android.internal.task.configureWith internal class DefaultUploadsService @AssistedInject constructor( @Assisted private val roomId: String, private val taskExecutor: TaskExecutor, - private val getUploadsTask: GetUploadsTask + private val getUploadsTask: GetUploadsTask, + private val cryptoService: CryptoService ) : UploadsService { @AssistedInject.Factory @@ -38,7 +40,7 @@ internal class DefaultUploadsService @AssistedInject constructor( override fun getUploads(numberOfEvents: Int, since: String?, callback: MatrixCallback): Cancelable { return getUploadsTask - .configureWith(GetUploadsTask.Params(roomId, numberOfEvents, since)) { + .configureWith(GetUploadsTask.Params(roomId, cryptoService.isRoomEncrypted(roomId), numberOfEvents, since)) { this.callback = callback } .executeBy(taskExecutor) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt index fa707c0bf8..7c0f46abff 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt @@ -17,12 +17,20 @@ package im.vector.matrix.android.internal.session.room.uploads import com.zhuinden.monarchy.Monarchy +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType import im.vector.matrix.android.api.session.events.model.toModel import im.vector.matrix.android.api.session.room.model.message.MessageContent import im.vector.matrix.android.api.session.room.model.message.MessageWithAttachmentContent +import im.vector.matrix.android.api.session.room.send.SendState import im.vector.matrix.android.api.session.room.sender.SenderInfo import im.vector.matrix.android.api.session.room.uploads.GetUploadsResult import im.vector.matrix.android.api.session.room.uploads.UploadEvent +import im.vector.matrix.android.internal.database.mapper.asDomain +import im.vector.matrix.android.internal.database.model.EventEntity +import im.vector.matrix.android.internal.database.model.EventEntityFields +import im.vector.matrix.android.internal.database.query.TimelineEventFilter +import im.vector.matrix.android.internal.database.query.whereType import im.vector.matrix.android.internal.network.executeRequest import im.vector.matrix.android.internal.session.filter.FilterFactory import im.vector.matrix.android.internal.session.room.RoomAPI @@ -38,6 +46,7 @@ internal interface GetUploadsTask : Task - val filter = FilterFactory.createUploadsFilter(params.numberOfEvents).toJSONString() - val chunk = executeRequest(eventBus) { - apiCall = roomAPI.getRoomMessagesFrom(params.roomId, since, PaginationDirection.BACKWARDS.value, params.numberOfEvents, filter) + if (params.isRoomEncrypted) { + // Get a chunk of events from cache for e2e rooms + + result = GetUploadsResult( + uploadEvents = emptyList(), + nextToken = "", + hasMore = false + ) + + var eventsFromRealm = emptyList() + monarchy.doWithRealm { realm -> + eventsFromRealm = EventEntity.whereType(realm, EventType.ENCRYPTED, params.roomId) + .like(EventEntityFields.DECRYPTION_RESULT_JSON, TimelineEventFilter.DecryptedContent.URL) + // FIXME Send event are stored twice in the DB. This is not normal. Keep only synced events + .like(EventEntityFields.SEND_STATE_STR, SendState.SYNCED.name) + .findAll() + .map { it.asDomain() } + // Exclude stickers + .filter { it.getClearType() != EventType.STICKER } + } + events = eventsFromRealm + } else { + val since = params.since ?: tokenStore.getLastToken() ?: throw IllegalStateException("No token available") + + val filter = FilterFactory.createUploadsFilter(params.numberOfEvents).toJSONString() + val chunk = executeRequest(eventBus) { + apiCall = roomAPI.getRoomMessagesFrom(params.roomId, since, PaginationDirection.BACKWARDS.value, params.numberOfEvents, filter) + } + + result = GetUploadsResult( + uploadEvents = emptyList(), + nextToken = chunk.end ?: "", + hasMore = chunk.hasMore() + ) + events = chunk.events } var uploadEvents = listOf() @@ -66,7 +108,7 @@ internal class DefaultGetUploadsTask @Inject constructor( monarchy.doWithRealm { realm -> val roomMemberHelper = RoomMemberHelper(realm, params.roomId) - uploadEvents = chunk.events.mapNotNull { event -> + uploadEvents = events.mapNotNull { event -> val eventId = event.eventId ?: return@mapNotNull null val messageContent = event.getClearContent()?.toModel() ?: return@mapNotNull null val messageWithAttachmentContent = (messageContent as? MessageWithAttachmentContent) ?: return@mapNotNull null @@ -91,10 +133,6 @@ internal class DefaultGetUploadsTask @Inject constructor( } } - return GetUploadsResult( - uploadEvents = uploadEvents, - nextToken = chunk.end ?: "", - hasMore = chunk.hasMore() - ) + return result.copy(uploadEvents = uploadEvents) } } From 6f804cab4dfa7260c798e11aa67e5673ba6ac5d1 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Wed, 27 May 2020 10:14:38 +0200 Subject: [PATCH 2/2] Avoid duplicated events in DB (one with localId and one with eventId from homeserver, once synced) --- .../database/helper/TimelineEventEntityHelper.kt | 9 +++++++++ .../android/internal/session/room/send/SendResponse.kt | 3 +++ .../internal/session/room/uploads/GetUploadsTask.kt | 3 --- .../android/internal/session/sync/RoomSyncHandler.kt | 3 +++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/TimelineEventEntityHelper.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/TimelineEventEntityHelper.kt index 0bf02aa92f..a46c49a706 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/TimelineEventEntityHelper.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/TimelineEventEntityHelper.kt @@ -18,6 +18,7 @@ package im.vector.matrix.android.internal.database.helper import im.vector.matrix.android.internal.database.model.TimelineEventEntity import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields +import im.vector.matrix.android.internal.extensions.assertIsManaged import io.realm.Realm internal fun TimelineEventEntity.Companion.nextId(realm: Realm): Long { @@ -28,3 +29,11 @@ internal fun TimelineEventEntity.Companion.nextId(realm: Realm): Long { currentIdNum.toLong() + 1 } } + +internal fun TimelineEventEntity.deleteOnCascade() { + assertIsManaged() + root?.deleteFromRealm() + annotations?.deleteFromRealm() + readReceipts?.deleteFromRealm() + deleteFromRealm() +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/SendResponse.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/SendResponse.kt index 947edee1ed..6d4784d225 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/SendResponse.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/SendResponse.kt @@ -21,5 +21,8 @@ import com.squareup.moshi.JsonClass @JsonClass(generateAdapter = true) internal data class SendResponse( + /** + * A unique identifier for the event. + */ @Json(name = "event_id") val eventId: String ) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt index 7c0f46abff..ecc7bc1b41 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/uploads/GetUploadsTask.kt @@ -22,7 +22,6 @@ import im.vector.matrix.android.api.session.events.model.EventType import im.vector.matrix.android.api.session.events.model.toModel import im.vector.matrix.android.api.session.room.model.message.MessageContent import im.vector.matrix.android.api.session.room.model.message.MessageWithAttachmentContent -import im.vector.matrix.android.api.session.room.send.SendState import im.vector.matrix.android.api.session.room.sender.SenderInfo import im.vector.matrix.android.api.session.room.uploads.GetUploadsResult import im.vector.matrix.android.api.session.room.uploads.UploadEvent @@ -76,8 +75,6 @@ internal class DefaultGetUploadsTask @Inject constructor( monarchy.doWithRealm { realm -> eventsFromRealm = EventEntity.whereType(realm, EventType.ENCRYPTED, params.roomId) .like(EventEntityFields.DECRYPTION_RESULT_JSON, TimelineEventFilter.DecryptedContent.URL) - // FIXME Send event are stored twice in the DB. This is not normal. Keep only synced events - .like(EventEntityFields.SEND_STATE_STR, SendState.SYNCED.name) .findAll() .map { it.asDomain() } // Exclude stickers diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt index 8c21d23a8c..a910944fbf 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt @@ -29,6 +29,7 @@ import im.vector.matrix.android.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult import im.vector.matrix.android.internal.database.helper.addOrUpdate import im.vector.matrix.android.internal.database.helper.addTimelineEvent +import im.vector.matrix.android.internal.database.helper.deleteOnCascade import im.vector.matrix.android.internal.database.mapper.ContentMapper import im.vector.matrix.android.internal.database.mapper.toEntity import im.vector.matrix.android.internal.database.model.ChunkEntity @@ -272,6 +273,8 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle event.mxDecryptionResult = adapter.fromJson(json) } } + // Finally delete the local echo + sendingEventEntity.deleteOnCascade() } else { Timber.v("Can't find corresponding local echo for tx:$it") }