From bfc70be5bb7fa1841c63f07c9b7ca2dfb3562715 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Thu, 15 Jul 2021 17:04:10 +0200 Subject: [PATCH] Record voice on Android 21 --- vector/build.gradle | 5 +- .../detail/composer/VoiceMessageHelper.kt | 91 +++++------------- .../features/voice/AbstractVoiceRecorder.kt | 95 +++++++++++++++++++ .../app/features/voice/VoiceRecorder.kt | 48 ++++++++++ .../app/features/voice/VoiceRecorderL.kt | 67 +++++++++++++ .../features/voice/VoiceRecorderProvider.kt | 33 +++++++ .../app/features/voice/VoiceRecorderQ.kt | 37 ++++++++ 7 files changed, 310 insertions(+), 66 deletions(-) create mode 100644 vector/src/main/java/im/vector/app/features/voice/AbstractVoiceRecorder.kt create mode 100644 vector/src/main/java/im/vector/app/features/voice/VoiceRecorder.kt create mode 100644 vector/src/main/java/im/vector/app/features/voice/VoiceRecorderL.kt create mode 100644 vector/src/main/java/im/vector/app/features/voice/VoiceRecorderProvider.kt create mode 100644 vector/src/main/java/im/vector/app/features/voice/VoiceRecorderQ.kt diff --git a/vector/build.gradle b/vector/build.gradle index f1b71741aa..9ae5ffd56e 100644 --- a/vector/build.gradle +++ b/vector/build.gradle @@ -144,7 +144,7 @@ android { buildConfigField "im.vector.app.features.crypto.keysrequest.OutboundSessionKeySharingStrategy", "outboundSessionKeySharingStrategy", "im.vector.app.features.crypto.keysrequest.OutboundSessionKeySharingStrategy.WhenTyping" - buildConfigField "Long", "VOICE_MESSAGE_DURATION_LIMIT_MS", "120000L" + buildConfigField "Long", "VOICE_MESSAGE_DURATION_LIMIT_MS", "120_000L" // If set, MSC3086 asserted identity messages sent on VoIP calls will cause the call to appear in the room corresponding to the asserted identity. // This *must* only be set in trusted environments. @@ -411,6 +411,9 @@ dependencies { // Passphrase strength helper implementation 'com.nulab-inc:zxcvbn:1.5.2' + // To convert voice message on old platforms + implementation 'com.arthenica:ffmpeg-kit-audio:4.4.LTS' + //Alerter implementation 'com.tapadoo.android:alerter:7.0.1' diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/composer/VoiceMessageHelper.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/composer/VoiceMessageHelper.kt index f202e0da56..4f07cf98fe 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/composer/VoiceMessageHelper.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/composer/VoiceMessageHelper.kt @@ -19,13 +19,13 @@ package im.vector.app.features.home.room.detail.composer import android.content.Context import android.media.AudioAttributes import android.media.MediaPlayer -import android.media.MediaRecorder -import android.os.Build import androidx.core.content.FileProvider import im.vector.app.BuildConfig import im.vector.app.core.utils.CountUpTimer import im.vector.app.features.home.room.detail.timeline.helper.VoiceMessagePlaybackTracker import im.vector.app.features.voice.VoiceFailure +import im.vector.app.features.voice.VoiceRecorder +import im.vector.app.features.voice.VoiceRecorderProvider import im.vector.lib.multipicker.entity.MultiPickerAudioType import im.vector.lib.multipicker.utils.toMultiPickerAudioType import org.matrix.android.sdk.api.extensions.orFalse @@ -34,7 +34,6 @@ import timber.log.Timber import java.io.File import java.io.FileInputStream import java.io.FileNotFoundException -import java.io.FileOutputStream import javax.inject.Inject /** @@ -42,54 +41,24 @@ import javax.inject.Inject */ class VoiceMessageHelper @Inject constructor( private val context: Context, - private val playbackTracker: VoiceMessagePlaybackTracker + private val playbackTracker: VoiceMessagePlaybackTracker, + voiceRecorderProvider: VoiceRecorderProvider ) { private var mediaPlayer: MediaPlayer? = null - private var mediaRecorder: MediaRecorder? = null - private val outputDirectory = File(context.cacheDir, "downloads") - private var outputFile: File? = null - private var lastRecordingFile: File? = null // In case of user pauses recording, plays another one in timeline + private var voiceRecorder: VoiceRecorder = voiceRecorderProvider.provideVoiceRecorder() private val amplitudeList = mutableListOf() private var amplitudeTicker: CountUpTimer? = null private var playbackTicker: CountUpTimer? = null - init { - if (!outputDirectory.exists()) { - outputDirectory.mkdirs() - } - } - - private fun initMediaRecorder() { - MediaRecorder().let { - it.setAudioSource(MediaRecorder.AudioSource.DEFAULT) - it.setOutputFormat(MediaRecorder.OutputFormat.OGG) - it.setAudioEncoder(MediaRecorder.AudioEncoder.OPUS) - it.setAudioEncodingBitRate(24000) - it.setAudioSamplingRate(48000) - mediaRecorder = it - } - } - fun startRecording() { stopPlayback() playbackTracker.makeAllPlaybacksIdle() - - outputFile = File(outputDirectory, "Voice message.ogg") - lastRecordingFile = outputFile amplitudeList.clear() try { - initMediaRecorder() - val mr = mediaRecorder!! - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - mr.setOutputFile(outputFile) - } else { - mr.setOutputFile(FileOutputStream(outputFile).fd) - } - mr.prepare() - mr.start() + voiceRecorder.startRecord() } catch (failure: Throwable) { throw VoiceFailure.UnableToRecord(failure) } @@ -97,9 +66,16 @@ class VoiceMessageHelper @Inject constructor( } fun stopRecording(): MultiPickerAudioType? { - internalStopRecording() + tryOrNull("Cannot stop media recording amplitude") { + stopRecordingAmplitudes() + } + val voiceMessageFile = tryOrNull("Cannot stop media recorder!") { + voiceRecorder.stopRecord() + voiceRecorder.getVoiceMessageFile() + } try { - outputFile?.let { + // TODO Improve this + voiceMessageFile?.let { val outputFileUri = FileProvider.getUriForFile(context, BuildConfig.APPLICATION_ID + ".fileProvider", it) return outputFileUri ?.toMultiPickerAudioType(context) @@ -113,38 +89,24 @@ class VoiceMessageHelper @Inject constructor( } } - private fun internalStopRecording() { + /** + * When entering in playback mode actually + */ + fun pauseRecording() { + voiceRecorder.stopRecord() + } + + fun deleteRecording() { tryOrNull("Cannot stop media recording amplitude") { stopRecordingAmplitudes() } tryOrNull("Cannot stop media recorder!") { - // Usually throws when the record is less than 1 second. - releaseMediaRecorder() + voiceRecorder.cancelRecord() } } - private fun releaseMediaRecorder() { - mediaRecorder?.let { - it.stop() - it.reset() - it.release() - } - - mediaRecorder = null - } - - fun pauseRecording() { - releaseMediaRecorder() - } - - fun deleteRecording() { - internalStopRecording() - outputFile?.delete() - outputFile = null - } - fun startOrPauseRecordingPlayback() { - lastRecordingFile?.let { + voiceRecorder.getCurrentRecord()?.let { startOrPausePlayback(VoiceMessagePlaybackTracker.RECORDING_ID, it) } } @@ -201,9 +163,8 @@ class VoiceMessageHelper @Inject constructor( } private fun onAmplitudeTick() { - val mr = mediaRecorder ?: return try { - val maxAmplitude = mr.maxAmplitude + val maxAmplitude = voiceRecorder.getMaxAmplitude() amplitudeList.add(maxAmplitude) playbackTracker.updateCurrentRecording(VoiceMessagePlaybackTracker.RECORDING_ID, amplitudeList) } catch (e: IllegalStateException) { diff --git a/vector/src/main/java/im/vector/app/features/voice/AbstractVoiceRecorder.kt b/vector/src/main/java/im/vector/app/features/voice/AbstractVoiceRecorder.kt new file mode 100644 index 0000000000..8a0f829f94 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voice/AbstractVoiceRecorder.kt @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voice + +import android.content.Context +import android.media.MediaRecorder +import android.os.Build +import java.io.File +import java.io.FileOutputStream + +abstract class AbstractVoiceRecorder( + context: Context, + private val filenameExt: String +) : VoiceRecorder { + private val outputDirectory = File(context.cacheDir, "voice_records") + + private var mediaRecorder: MediaRecorder? = null + private var outputFile: File? = null + + init { + if (!outputDirectory.exists()) { + outputDirectory.mkdirs() + } + } + + abstract fun setOutputFormat(mediaRecorder: MediaRecorder) + abstract fun convertFile(recordedFile: File?): File? + + private fun init() { + MediaRecorder().let { + it.setAudioSource(MediaRecorder.AudioSource.DEFAULT) + setOutputFormat(it) + it.setAudioEncodingBitRate(24000) + it.setAudioSamplingRate(48000) + mediaRecorder = it + } + } + + override fun startRecord() { + init() + outputFile = File(outputDirectory, "Voice message.$filenameExt") + + val mr = mediaRecorder ?: return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + mr.setOutputFile(outputFile) + } else { + mr.setOutputFile(FileOutputStream(outputFile).fd) + } + mr.prepare() + mr.start() + } + + override fun stopRecord() { + // Can throw when the record is less than 1 second. + mediaRecorder?.let { + it.stop() + it.reset() + it.release() + } + mediaRecorder = null + } + + override fun cancelRecord() { + stopRecord() + + outputFile?.delete() + outputFile = null + } + + override fun getMaxAmplitude(): Int { + return mediaRecorder?.maxAmplitude ?: 0 + } + + override fun getCurrentRecord(): File? { + return outputFile + } + + override fun getVoiceMessageFile(): File? { + return convertFile(outputFile) + } +} diff --git a/vector/src/main/java/im/vector/app/features/voice/VoiceRecorder.kt b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorder.kt new file mode 100644 index 0000000000..17e70997b2 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorder.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voice + +import java.io.File + +interface VoiceRecorder { + /** + * Start the recording + */ + fun startRecord() + + /** + * Stop the recording + */ + fun stopRecord() + + /** + * Remove the file + */ + fun cancelRecord() + + fun getMaxAmplitude(): Int + + /** + * Not guaranteed to be a ogg file + */ + fun getCurrentRecord(): File? + + /** + * Guaranteed to be a ogg file + */ + fun getVoiceMessageFile(): File? +} diff --git a/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderL.kt b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderL.kt new file mode 100644 index 0000000000..2d40f5f7a3 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderL.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voice + +import android.content.Context +import android.media.MediaRecorder +import com.arthenica.ffmpegkit.FFmpegKit +import com.arthenica.ffmpegkit.FFmpegKitConfig +import com.arthenica.ffmpegkit.Level +import com.arthenica.ffmpegkit.ReturnCode +import im.vector.app.BuildConfig +import timber.log.Timber +import java.io.File + +class VoiceRecorderL(context: Context) : AbstractVoiceRecorder(context, "mp4") { + override fun setOutputFormat(mediaRecorder: MediaRecorder) { + // Use AAC/MP4 format here + mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) + mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC) + } + + override fun convertFile(recordedFile: File?): File? { + if (BuildConfig.DEBUG) { + FFmpegKitConfig.setLogLevel(Level.AV_LOG_INFO) + } + recordedFile ?: return null + // Convert to OGG + val targetFile = File(recordedFile.path.removeSuffix("mp4") + "ogg") + if (targetFile.exists()) { + targetFile.delete() + } + val start = System.currentTimeMillis() + val session = FFmpegKit.execute("-i \"${recordedFile.path}\" -c:a libvorbis \"${targetFile.path}\"") + val duration = System.currentTimeMillis() - start + Timber.d("Convert to ogg in $duration ms. Size in bytes from ${recordedFile.length()} to ${targetFile.length()}") + return when { + ReturnCode.isSuccess(session.returnCode) -> { + // SUCCESS + targetFile + } + ReturnCode.isCancel(session.returnCode) -> { + // CANCEL + null + } + else -> { + // FAILURE + Timber.e("Command failed with state ${session.state} and rc ${session.returnCode}.${session.failStackTrace}") + // TODO throw? + null + } + } + } +} diff --git a/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderProvider.kt b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderProvider.kt new file mode 100644 index 0000000000..004d520a6f --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderProvider.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voice + +import android.content.Context +import android.os.Build +import javax.inject.Inject + +class VoiceRecorderProvider @Inject constructor( + private val context: Context +) { + fun provideVoiceRecorder(): VoiceRecorder { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + VoiceRecorderQ(context) + } else { + VoiceRecorderL(context) + } + } +} diff --git a/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderQ.kt b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderQ.kt new file mode 100644 index 0000000000..d6f4676893 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/voice/VoiceRecorderQ.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.voice + +import android.content.Context +import android.media.MediaRecorder +import android.os.Build +import androidx.annotation.RequiresApi +import java.io.File + +@RequiresApi(Build.VERSION_CODES.Q) +class VoiceRecorderQ(context: Context) : AbstractVoiceRecorder(context, "ogg") { + override fun setOutputFormat(mediaRecorder: MediaRecorder) { + // We can directly use OGG here + mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.OGG) + mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.OPUS) + } + + override fun convertFile(recordedFile: File?): File? { + // Nothing to do here + return recordedFile + } +}