diff --git a/app/src/main/java/org/session/libsession/messaging/messages/Message.kt b/app/src/main/java/org/session/libsession/messaging/messages/Message.kt index 077899dc34..dde3786da3 100644 --- a/app/src/main/java/org/session/libsession/messaging/messages/Message.kt +++ b/app/src/main/java/org/session/libsession/messaging/messages/Message.kt @@ -58,6 +58,7 @@ abstract class Message { val Message.senderOrSync get() = when(this) { is VisibleMessage -> syncTarget ?: sender!! is ExpirationTimerUpdate -> syncTarget ?: sender!! + is UnsupportedMessage -> syncTarget ?: sender!! else -> sender!! } } diff --git a/app/src/main/java/org/session/libsession/messaging/messages/UnsupportedMessage.kt b/app/src/main/java/org/session/libsession/messaging/messages/UnsupportedMessage.kt new file mode 100644 index 0000000000..fd7c971127 --- /dev/null +++ b/app/src/main/java/org/session/libsession/messaging/messages/UnsupportedMessage.kt @@ -0,0 +1,53 @@ +package org.session.libsession.messaging.messages + +import org.session.libsession.database.MessageDataProvider +import org.session.protos.SessionProtos + +/** + * A swarm message this client positively identified as something it can't process. There are only + * two ways to get one: + * - [Kind.NEWER_FORMAT]: the raw one-to-one swarm data starts with `0x00`, identified without + * attempting decryption. It has no sender and no conversation. + * - [Kind.UNKNOWN_TYPE]: the message decrypted (so the sender is authenticated) but its `Content` + * holds a type this client doesn't know. + * + * Receive-only: it is never sent. + */ +class UnsupportedMessage( + val kind: Kind, + val placement: Placement, + val rawData: ByteArray, + val swarmPublicKey: String, + val namespace: Int, + val serverTimestampMs: Long, + val serverExpiryMs: Long?, + val syncTarget: String?, +) : Message() { + + /** The values are stored in the shared `unsupported_message.kind` column, so must not change. */ + enum class Kind(val dbValue: String) { + NEWER_FORMAT("newerFormat"), + UNKNOWN_TYPE("unknownType"); + + companion object { + fun fromDbValue(value: String): Kind? = entries.firstOrNull { it.dbValue == value } + } + } + + enum class Placement { + /** Nowhere to show it (a newer-format message, or our own sync without a target), only retained */ + NONE, + INCOMING, + OUTGOING, + } + + override val isSelfSendValid: Boolean get() = true + + override fun isValid(): Boolean = super.isValid() && rawData.isNotEmpty() && !serverHash.isNullOrEmpty() + + override fun shouldDiscardIfBlocked(): Boolean = true + + override fun buildProto(builder: SessionProtos.Content.Builder, messageDataProvider: MessageDataProvider) { + throw UnsupportedOperationException("An UnsupportedMessage is never sent") + } +} diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/MessageParser.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/MessageParser.kt index fbf3a7aa71..dd95ae8f95 100644 --- a/app/src/main/java/org/session/libsession/messaging/sending_receiving/MessageParser.kt +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/MessageParser.kt @@ -1,5 +1,7 @@ package org.session.libsession.messaging.sending_receiving +import androidx.annotation.VisibleForTesting +import network.loki.messenger.libsession_util.Namespace import network.loki.messenger.libsession_util.SessionEncrypt import network.loki.messenger.libsession_util.pro.ProProof import network.loki.messenger.libsession_util.protocol.DecodedEnvelope @@ -8,6 +10,8 @@ import network.loki.messenger.libsession_util.protocol.SessionProtocol import network.loki.messenger.libsession_util.util.asSequence import org.session.libsession.database.StorageProtocol import org.session.libsession.messaging.messages.Message +import org.session.libsession.messaging.messages.UnsupportedMessage +import org.session.libsession.messaging.messages.copyExpiration import org.session.libsession.messaging.messages.control.CallMessage import org.session.libsession.messaging.messages.control.DataExtractionNotification import org.session.libsession.messaging.messages.control.ExpirationTimerUpdate @@ -58,6 +62,19 @@ class MessageParser @Inject constructor( val pro: DecodedPro? ) + /** + * Where a swarm message came from, exactly as retrieved, so an [UnsupportedMessage] can be + * retained in a form that can be replayed through this parser later. + */ + class SwarmOrigin( + val rawData: ByteArray, + val swarmPublicKey: String, + val namespace: Int, + val serverHash: String, + val serverTimestampMs: Long, + val serverExpiryMs: Long?, + ) + private fun createMessageFromProto(proto: SessionProtos.Content, isGroupMessage: Boolean): Message { val message = ReadReceipt.fromProto(proto) ?: @@ -84,7 +101,8 @@ class MessageParser @Inject constructor( isForGroup: Boolean, currentUserId: AccountId, currentUserBlindedIDs: List, - senderIdPrefix: IdPrefix + senderIdPrefix: IdPrefix, + swarmOrigin: SwarmOrigin?, ): ParseResult { return parseMessage( sender = AccountId(senderIdPrefix, decodedEnvelope.senderX25519PubKey.data), @@ -96,6 +114,7 @@ class MessageParser @Inject constructor( isForGroup = isForGroup, currentUserId = currentUserId, currentUserBlindedIDs = currentUserBlindedIDs, + swarmOrigin = swarmOrigin, ) } @@ -109,6 +128,7 @@ class MessageParser @Inject constructor( isForGroup: Boolean, currentUserId: AccountId, currentUserBlindedIDs: List, + swarmOrigin: SwarmOrigin? = null, ): ParseResult { val proto = SessionProtos.Content.parseFrom(contentPlaintext) @@ -122,6 +142,30 @@ class MessageParser @Inject constructor( } } + // Only swarm messages are considered: communities and their inboxes are open to anyone, so a + // placeholder there would invite spam more than it informs. + if (swarmOrigin != null && + UnsupportedMessageDetection.isUnknownType(contentPlaintext) { + hasValidKnownContent( + proto = proto, + isForGroup = isForGroup, + sender = sender, + currentUserId = currentUserId, + messageTimestampMs = messageTimestampMs, + ) + } + ) { + return parseUnknownTypeMessage( + proto = proto, + sender = sender, + messageTimestampMs = messageTimestampMs, + checkForBlockStatus = checkForBlockStatus, + isForGroup = isForGroup, + currentUserId = currentUserId, + swarmOrigin = swarmOrigin, + ) + } + val message = createMessageFromProto(proto, isGroupMessage = isForGroup) // Blocked sender check @@ -163,10 +207,7 @@ class MessageParser @Inject constructor( } // Validate - var isValid = message.isValid() - // TODO: Legacy code: why this is check needed? - if (message is VisibleMessage && !isValid && proto.dataMessage.attachmentsCount != 0) { isValid = true } - if (!isValid) { + if (!isValidMessage(message, proto)) { throw NonRetryableException("Invalid message") } @@ -190,12 +231,106 @@ class MessageParser @Inject constructor( } + private fun isValidMessage(message: Message, proto: SessionProtos.Content): Boolean { + // TODO: Legacy code: why this is check needed? + return message.isValid() || + (message is VisibleMessage && proto.dataMessage.attachmentsCount != 0) + } + + @VisibleForTesting + internal fun hasValidKnownContent( + proto: SessionProtos.Content, + isForGroup: Boolean, + sender: AccountId, + currentUserId: AccountId, + messageTimestampMs: Long, + ): Boolean { + val message = runCatching { createMessageFromProto(proto, isGroupMessage = isForGroup) } + .getOrNull() + ?: return false + + message.sender = sender.hexString + message.recipient = currentUserId.hexString + message.sentTimestamp = messageTimestampMs + message.receivedTimestamp = snodeClock.currentTimeMillis() + + return isValidMessage(message, proto) + } + + @VisibleForTesting + internal fun parseUnknownTypeMessage( + proto: SessionProtos.Content, + sender: AccountId, + messageTimestampMs: Long, + checkForBlockStatus: Boolean, + isForGroup: Boolean, + currentUserId: AccountId, + swarmOrigin: SwarmOrigin, + ): ParseResult { + val isSenderSelf = sender == currentUserId + + if (checkForBlockStatus && !isSenderSelf && isUserBlocked(sender)) { + throw NonRetryableException("Sender($sender) is blocked from sending message to us") + } + + val (placement, syncTarget) = UnsupportedMessageDetection.placement( + isGroup = isForGroup, + isSenderSelf = isSenderSelf, + syncTarget = proto.dataMessage.takeIf { proto.hasDataMessage() && it.hasSyncTarget() }?.syncTarget, + ) + + val message = UnsupportedMessage( + kind = UnsupportedMessage.Kind.UNKNOWN_TYPE, + placement = placement, + rawData = swarmOrigin.rawData, + swarmPublicKey = swarmOrigin.swarmPublicKey, + namespace = swarmOrigin.namespace, + serverTimestampMs = swarmOrigin.serverTimestampMs, + serverExpiryMs = swarmOrigin.serverExpiryMs, + syncTarget = syncTarget, + ).copyExpiration(proto).apply { + this.sender = sender.hexString + recipient = currentUserId.hexString + sentTimestamp = messageTimestampMs + receivedTimestamp = snodeClock.currentTimeMillis() + this.isSenderSelf = isSenderSelf + serverHash = swarmOrigin.serverHash + } + + if (!message.isValid()) { + throw NonRetryableException("Invalid message") + } + + return ParseResult(message = message, proto = proto, pro = null) + } + fun parse1o1Message( data: ByteArray, serverHash: String?, currentUserEd25519PrivKey: ByteArray, currentUserId: AccountId, + serverTimestampMs: Long, + serverExpiryMs: Long?, ): ParseResult { + // A message in a newer format is recognised from its first byte without decrypting it. A + // message that merely fails to decrypt deliberately gets nothing at all: it can't be + // attributed to anyone, so it could be spam, corruption or an attacker, and anything we + // showed for it would claim that someone really sent us something. Even a newer-format + // message only gets an account-level banner, never a bubble, because the format's prefix + // is unauthenticated and its sender is inside the encryption. + if (UnsupportedMessageDetection.isNewerFormat(data)) { + return parseNewerFormatMessage( + data = data, + serverHash = serverHash + // Only push without metadata lacks a hash, and the poller fetches the same + // message again with one. + ?: throw NonRetryableException("Newer format message without a hash"), + currentUserId = currentUserId, + serverTimestampMs = serverTimestampMs, + serverExpiryMs = serverExpiryMs, + ) + } + val envelop = SessionProtocol.decodeFor1o1( myEd25519PrivKey = currentUserEd25519PrivKey, payload = data, @@ -210,17 +345,63 @@ class MessageParser @Inject constructor( senderIdPrefix = IdPrefix.STANDARD, currentUserId = currentUserId, currentUserBlindedIDs = emptyList(), + swarmOrigin = serverHash?.let { + SwarmOrigin( + rawData = data, + swarmPublicKey = currentUserId.hexString, + namespace = Namespace.DEFAULT(), + serverHash = it, + serverTimestampMs = serverTimestampMs, + serverExpiryMs = serverExpiryMs, + ) + }, ).also { result -> result.message.serverHash = serverHash } } + private fun parseNewerFormatMessage( + data: ByteArray, + serverHash: String, + currentUserId: AccountId, + serverTimestampMs: Long, + serverExpiryMs: Long?, + ): ParseResult { + val message = UnsupportedMessage( + kind = UnsupportedMessage.Kind.NEWER_FORMAT, + placement = UnsupportedMessage.Placement.NONE, + rawData = data, + swarmPublicKey = currentUserId.hexString, + namespace = Namespace.DEFAULT(), + serverTimestampMs = serverTimestampMs, + serverExpiryMs = serverExpiryMs, + syncTarget = null, + ).apply { + // The real sender is inside the encryption; the current user stands in so nothing + // downstream treats it as having come from someone else. + sender = currentUserId.hexString + recipient = currentUserId.hexString + sentTimestamp = serverTimestampMs + receivedTimestamp = snodeClock.currentTimeMillis() + isSenderSelf = true + this.serverHash = serverHash + } + + if (!message.isValid()) { + throw NonRetryableException("Invalid message") + } + + return ParseResult(message = message, proto = SessionProtos.Content.getDefaultInstance(), pro = null) + } + fun parseGroupMessage( data: ByteArray, serverHash: String, groupId: AccountId, currentUserEd25519PrivKey: ByteArray, currentUserId: AccountId, + serverTimestampMs: Long, + serverExpiryMs: Long?, ): ParseResult { val keys = configFactory.withGroupConfigs(groupId) { it.groupKeys.groupKeys() @@ -242,6 +423,14 @@ class MessageParser @Inject constructor( senderIdPrefix = IdPrefix.STANDARD, currentUserId = currentUserId, currentUserBlindedIDs = emptyList(), + swarmOrigin = SwarmOrigin( + rawData = data, + swarmPublicKey = groupId.hexString, + namespace = Namespace.GROUP_MESSAGES(), + serverHash = serverHash, + serverTimestampMs = serverTimestampMs, + serverExpiryMs = serverExpiryMs, + ), ).also { result -> result.message.serverHash = serverHash } diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageProcessor.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageProcessor.kt index c47ccd57df..236bfdcafe 100644 --- a/app/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageProcessor.kt +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageProcessor.kt @@ -15,6 +15,7 @@ import org.session.libsession.database.MessageDataProvider import org.session.libsession.database.userAuth import org.session.libsession.messaging.messages.Message import org.session.libsession.messaging.messages.Message.Companion.senderOrSync +import org.session.libsession.messaging.messages.UnsupportedMessage import org.session.libsession.messaging.messages.control.CallMessage import org.session.libsession.messaging.messages.control.DataExtractionNotification import org.session.libsession.messaging.messages.control.ExpirationTimerUpdate @@ -51,6 +52,7 @@ import org.thoughtcrime.securesms.database.BlindMappingRepository import org.thoughtcrime.securesms.database.RecipientRepository import org.thoughtcrime.securesms.database.Storage import org.thoughtcrime.securesms.database.ThreadDatabase +import org.thoughtcrime.securesms.database.UnsupportedMessageDatabase import org.thoughtcrime.securesms.database.getOrCreateThreadIdFor import org.thoughtcrime.securesms.database.model.MessageId import org.thoughtcrime.securesms.database.model.ReactionRecord @@ -82,7 +84,9 @@ class ReceivedMessageProcessor @Inject constructor( private val blindMappingRepository: BlindMappingRepository, private val messageParser: MessageParser, private val swarmApiExecutor: SwarmApiExecutor, - private val deleteMessageApiFactory: DeleteMessageApi.Factory + private val deleteMessageApiFactory: DeleteMessageApi.Factory, + private val unsupportedMessageHandler: Provider, + private val unsupportedMessageDatabase: UnsupportedMessageDatabase, ) { private val threadMutexes = ConcurrentHashMap() @@ -132,6 +136,14 @@ class ReceivedMessageProcessor @Inject constructor( proto: SessionProtos.Content, pro: DecodedPro?, ) = withThreadLock(threadAddress) { + // Retained even when there's no conversation to show it in, or the conversation is hidden, + // so it can't wait for the hidden-contact discard or the thread lookup below. The handler + // never un-hides a conversation; it only leaves out the placeholder. + if (message is UnsupportedMessage) { + unsupportedMessageHandler.get().handle(context, threadAddress, message) + return@withThreadLock + } + // The logic to check if the message should be discarded due to being from a hidden contact. if (threadAddress is Address.Standard && message.sentTimestamp != null && @@ -444,6 +456,10 @@ class ReceivedMessageProcessor @Inject constructor( val timestamp = message.timestamp ?: return null val author = message.author ?: return null + + // Before the message lookup, which finds nothing for a retained message with no placeholder + unsupportedMessageDatabase.deleteForUnsend(sender = author, sentTimestampMs = timestamp) + val messageToDelete = storage.getMessageByTimestamp(timestamp, author, false) ?: return null val messageIdToDelete = messageToDelete.messageId val messageType = messageToDelete.individualRecipient?.getType() diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBanner.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBanner.kt new file mode 100644 index 0000000000..4ba00c5fd9 --- /dev/null +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBanner.kt @@ -0,0 +1,96 @@ +package org.session.libsession.messaging.sending_receiving + +import androidx.annotation.StringRes +import network.loki.messenger.R +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onStart +import org.thoughtcrime.securesms.preferences.PreferenceKey +import org.thoughtcrime.securesms.preferences.PreferenceStorage +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The account-level banner for unsupported messages that can't be shown in any conversation. + * + * It deliberately carries no count and makes no claim about senders: a stranger can deposit + * newer-format data in our one-to-one namespace, so the banner has to stay true when they do. + */ +object UnsupportedMessageBanner { + /** + * After a dismissal the banner only reappears for a trigger at least this long afterwards, + * which bounds how often a stranger can bring it back. + */ + const val REAPPEAR_AFTER_DISMISSAL_MS: Long = 7L * 24 * 60 * 60 * 1000 + + enum class Trigger { + NEWER_FORMAT, + + /** An unknown type from one of our own devices that couldn't be placed in a conversation */ + OTHER_DEVICE, + } + + enum class State(@StringRes val textRes: Int?) { + HIDDEN(null), + GENERAL(R.string.messageUnsupportedBanner), + OTHER_DEVICE(R.string.messageUnsupportedBannerLinkedDevice), + } + + fun state(triggeredAtMs: Long?, otherDeviceTriggeredAtMs: Long?, dismissedAtMs: Long?): State { + if (triggeredAtMs == null) return State.HIDDEN + + // Without a dismissal there's no threshold at all. Defaulting it to Long.MIN_VALUE instead + // would make a missing other-device trigger compare as "after" it and pick the wrong text. + val visibleFromMs = dismissedAtMs?.let { it + REAPPEAR_AFTER_DISMISSAL_MS } + + if (visibleFromMs != null && triggeredAtMs < visibleFromMs) return State.HIDDEN + + val otherDeviceIsCurrent = otherDeviceTriggeredAtMs != null && + (visibleFromMs == null || otherDeviceTriggeredAtMs >= visibleFromMs) + + return if (otherDeviceIsCurrent) State.OTHER_DEVICE else State.GENERAL + } +} + +@Singleton +class UnsupportedMessageBannerStore @Inject constructor( + private val prefs: PreferenceStorage, +) { + fun recordTrigger(trigger: UnsupportedMessageBanner.Trigger, nowMs: Long) { + prefs[TRIGGERED_AT_MS] = nowMs + + if (trigger == UnsupportedMessageBanner.Trigger.OTHER_DEVICE) { + prefs[OTHER_DEVICE_TRIGGERED_AT_MS] = nowMs + } + } + + fun dismiss(nowMs: Long) { + prefs[DISMISSED_AT_MS] = nowMs + } + + fun currentState(): UnsupportedMessageBanner.State = UnsupportedMessageBanner.state( + triggeredAtMs = prefs[TRIGGERED_AT_MS].takeIf { it != UNSET }, + otherDeviceTriggeredAtMs = prefs[OTHER_DEVICE_TRIGGERED_AT_MS].takeIf { it != UNSET }, + dismissedAtMs = prefs[DISMISSED_AT_MS].takeIf { it != UNSET }, + ) + + val state: Flow + get() = prefs.changes() + .filter { it.name in KEY_NAMES } + .map { } + .onStart { emit(Unit) } + .map { currentState() } + .distinctUntilChanged() + + companion object { + private const val UNSET = -1L + + private val TRIGGERED_AT_MS = PreferenceKey.long("unsupported_message_banner_triggered_at_ms", UNSET) + private val OTHER_DEVICE_TRIGGERED_AT_MS = PreferenceKey.long("unsupported_message_banner_other_device_triggered_at_ms", UNSET) + private val DISMISSED_AT_MS = PreferenceKey.long("unsupported_message_banner_dismissed_at_ms", UNSET) + + private val KEY_NAMES = setOf(TRIGGERED_AT_MS.name, OTHER_DEVICE_TRIGGERED_AT_MS.name, DISMISSED_AT_MS.name) + } +} diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetection.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetection.kt new file mode 100644 index 0000000000..508586dd3a --- /dev/null +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetection.kt @@ -0,0 +1,149 @@ +package org.session.libsession.messaging.sending_receiving + +import org.session.libsession.messaging.messages.UnsupportedMessage +import org.session.libsignal.utilities.AccountId +import org.session.libsignal.utilities.IdPrefix + +/** + * The decisions behind [UnsupportedMessage], kept free of libsession and database access so they + * can be unit tested (libsession's native library can't be loaded in JVM tests). + * + * The table and rules here are shared with the iOS and Desktop clients, so a later client (or a + * database import) can treat all three the same way. + */ +object UnsupportedMessageDetection { + /** + * The highest top-level `Content` field number ever assigned, including retired numbers and + * metadata such as `msgId = 18` which newer clients add alongside a type we do know. Comparing + * against this bound rather than our own schema stops that metadata from looking like a new type. + */ + const val HIGHEST_KNOWN_CONTENT_FIELD_NUMBER = 18 + + private const val WIRE_TYPE_VARINT = 0 + private const val WIRE_TYPE_FIXED64 = 1 + private const val WIRE_TYPE_LENGTH_DELIMITED = 2 + private const val WIRE_TYPE_START_GROUP = 3 + private const val WIRE_TYPE_END_GROUP = 4 + private const val WIRE_TYPE_FIXED32 = 5 + + /** + * No v1 one-to-one message can start with a zero byte (it's protobuf, and field number 0 is + * invalid), which is exactly why the newer wire format starts with one. + * + * Only meaningful for the one-to-one namespace: other namespaces are encrypted with symmetric + * keys so their data can start with any byte. + */ + fun isNewerFormat(rawData: ByteArray): Boolean = rawData.isNotEmpty() && rawData[0] == 0.toByte() + + /** + * Returns the field numbers of the top-level fields of a serialised protobuf message, or null if + * the data isn't well-formed. + * + * Hand-written rather than relying on the protobuf library's unknown-field set so the result + * doesn't depend on how (or whether) the generated classes retain unknown fields. + */ + fun topLevelFieldNumbers(data: ByteArray): List? { + var index = 0 + val result = mutableListOf() + + fun readVarint(): Long? { + var value = 0L + var shift = 0 + + while (index < data.size && shift < 64) { + val byte = data[index].toInt() and 0xFF + index++ + value = value or ((byte and 0x7F).toLong() shl shift) + + if (byte and 0x80 == 0) return value + + shift += 7 + } + + return null + } + + while (index < data.size) { + val key = readVarint() ?: return null + val fieldNumber = key ushr 3 + + if (fieldNumber <= 0 || fieldNumber > Int.MAX_VALUE) return null + + // A key is `(fieldNumber shl 3) or wireType`. The switch is on the wire type, which says + // how many bytes the value occupies, not on the field number. + when ((key and 0x7).toInt()) { + WIRE_TYPE_VARINT -> readVarint() ?: return null + WIRE_TYPE_FIXED64 -> index += 8 + WIRE_TYPE_LENGTH_DELIMITED -> { + val length = readVarint() ?: return null + if (length < 0 || length > (data.size - index)) return null + index += length.toInt() + } + WIRE_TYPE_FIXED32 -> index += 4 + // Groups are deprecated and unused by Session, and 6 and 7 are undefined, so + // anything else is malformed + WIRE_TYPE_START_GROUP, WIRE_TYPE_END_GROUP -> return null + else -> return null + } + + if (index > data.size) return null + + result += fieldNumber.toInt() + } + + return result + } + + fun containsUnknownContentType(contentPlaintext: ByteArray): Boolean = + topLevelFieldNumbers(contentPlaintext).orEmpty().any { it > HIGHEST_KNOWN_CONTENT_FIELD_NUMBER } + + /** + * Positive evidence of a type added after this client was built. Requiring known content to also + * be invalid matters for sync copies: the sender adds `dataMessage.syncTarget` to a sync copy + * whatever its type, so a newer type arrives as an otherwise-empty (and invalid) data message. + * + * [hasValidKnownContent] is only evaluated once an unknown field is found, which is rare, so + * ordinary messages aren't turned into a message twice. + */ + inline fun isUnknownType(contentPlaintext: ByteArray, hasValidKnownContent: () -> Boolean): Boolean = + containsUnknownContentType(contentPlaintext) && !hasValidKnownContent() + + /** + * @return the placement and, for an outgoing one-to-one sync, the conversation it belongs in. + */ + fun placement( + isGroup: Boolean, + isSenderSelf: Boolean, + syncTarget: String?, + ): Pair = when { + isGroup -> (if (isSenderSelf) UnsupportedMessage.Placement.OUTGOING else UnsupportedMessage.Placement.INCOMING) to null + !isSenderSelf -> UnsupportedMessage.Placement.INCOMING to null + else -> syncTarget + ?.takeIf { AccountId.fromStringOrNull(it)?.prefix == IdPrefix.STANDARD } + ?.let { UnsupportedMessage.Placement.OUTGOING to it } + ?: (UnsupportedMessage.Placement.NONE to null) + } + + /** + * When a retained message has no placeholder there is nothing to own its expiry, so work out when + * the record itself should go. + * + * A disappear-after-send setting is readable from an unknown type. Disappear-after-read can never + * start because the message is never shown, so it falls to the swarm expiry, which only signals a + * disappearing message when it's shorter than the default TTL (after-send senders store with a + * TTL matching their timer; everything else gets the default and is kept for the importer). + */ + fun retainedExpiryMs( + afterSendStartedAtMs: Long?, + afterSendDurationMs: Long?, + serverTimestampMs: Long, + serverExpiryMs: Long?, + defaultTtlMs: Long, + ): Long? { + if (afterSendStartedAtMs != null && afterSendDurationMs != null && afterSendDurationMs > 0) { + return afterSendStartedAtMs + afterSendDurationMs + } + + return serverExpiryMs?.takeIf { (it - serverTimestampMs) < defaultTtlMs } + } +} diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageHandler.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageHandler.kt new file mode 100644 index 0000000000..3e67013304 --- /dev/null +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageHandler.kt @@ -0,0 +1,188 @@ +package org.session.libsession.messaging.sending_receiving + +import network.loki.messenger.BuildConfig +import network.loki.messenger.libsession_util.PRIORITY_HIDDEN +import network.loki.messenger.libsession_util.util.ExpiryMode +import org.session.libsession.messaging.messages.Message +import org.session.libsession.messaging.messages.UnsupportedMessage +import org.session.libsession.messaging.messages.signal.IncomingMediaMessage +import org.session.libsession.messaging.messages.signal.OutgoingMediaMessage +import org.session.libsession.network.SnodeClock +import org.session.libsession.snode.SnodeMessage +import org.session.libsession.utilities.Address +import org.session.libsession.utilities.Address.Companion.toAddress +import org.session.libsession.utilities.MessageExpirationManagerProtocol +import org.session.libsignal.utilities.Log +import org.thoughtcrime.securesms.database.LokiMessageDatabase +import org.thoughtcrime.securesms.database.MmsDatabase +import org.thoughtcrime.securesms.database.Storage +import org.thoughtcrime.securesms.database.UnsupportedMessageDatabase +import org.thoughtcrime.securesms.database.model.MessageId +import org.thoughtcrime.securesms.database.model.content.UnsupportedMessageContent +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +@Singleton +class UnsupportedMessageHandler @Inject constructor( + private val storage: Storage, + private val mmsDatabase: MmsDatabase, + private val lokiMessageDatabase: LokiMessageDatabase, + private val unsupportedMessageDatabase: UnsupportedMessageDatabase, + private val bannerStore: UnsupportedMessageBannerStore, + private val messageExpirationManager: Provider, + private val snodeClock: SnodeClock, +) { + /** + * Retains [message] and, when it belongs in a conversation that already exists, inserts a + * placeholder for it there. + */ + fun handle( + context: ReceivedMessageProcessor.MessageProcessingContext, + threadAddress: Address.Conversable, + message: UnsupportedMessage, + ) { + val serverHash = requireNotNull(message.serverHash) { "An UnsupportedMessage always has a hash" } + + if (unsupportedMessageDatabase.exists(serverHash)) return + + // Never create a conversation (or a message request) for something we can't show, nor put + // one in a hidden conversation: visibility is config-synced, so anything that un-hid it + // would un-hide it on every device. The record is still retained either way so a newer + // client can recover it. + val threadId = when { + message.placement == UnsupportedMessage.Placement.NONE -> null + context.getThreadRecipient(threadAddress).priority == PRIORITY_HIDDEN -> null + else -> context.threadIDs[threadAddress] + ?: storage.getThreadId(threadAddress)?.also { context.threadIDs[threadAddress] = it } + } + + val placeholderId = threadId?.let { insertPlaceholder(context, it, threadAddress, message) } + val nowMs = snodeClock.currentTimeMillis() + + unsupportedMessageDatabase.insert( + kind = message.kind, + swarmPublicKey = message.swarmPublicKey, + namespace = message.namespace, + hash = serverHash, + // A newer-format message's sender and sent timestamp are inside the encryption; the + // message carries stand-ins for them which must not match an unsend request. + sender = message.sender.takeIf { message.kind == UnsupportedMessage.Kind.UNKNOWN_TYPE }, + sentTimestampMs = message.sentTimestamp.takeIf { message.kind == UnsupportedMessage.Kind.UNKNOWN_TYPE }, + serverTimestampMs = message.serverTimestampMs, + serverExpiryMs = message.serverExpiryMs, + data = message.rawData, + placeholderMessageId = placeholderId?.id, + // A placeholder owns the expiry through the normal disappearing-messages path + expiresAtMs = if (placeholderId != null) null else retainedExpiryMs( + message = message, + serverTimestampMs = message.serverTimestampMs, + serverExpiryMs = message.serverExpiryMs, + ), + receivedAtMs = nowMs, + lastAttemptVersion = currentVersion(), + ) + + // A failure here must not fail the receive: the limits are enforced again at the next + // launch or foreground. Expired rows are left to that run too. + try { + unsupportedMessageDatabase.enforceRetentionCaps() + } catch (e: Exception) { + Log.e(TAG, "Failed to enforce retained message limits", e) + } + + Log.i(TAG, "Retained ${message.kind.dbValue} message $serverHash (${message.rawData.size} bytes), placeholder: ${placeholderId != null}") + + // Only things which say this device is behind, and which a stranger can't use to make a + // claim about who sent what, raise the banner. An unknown type from someone else either has + // a placeholder or came from a stranger. + when { + message.kind == UnsupportedMessage.Kind.NEWER_FORMAT -> + bannerStore.recordTrigger(UnsupportedMessageBanner.Trigger.NEWER_FORMAT, nowMs) + + message.placement == UnsupportedMessage.Placement.NONE && message.isSenderSelf -> + bannerStore.recordTrigger(UnsupportedMessageBanner.Trigger.OTHER_DEVICE, nowMs) + } + } + + private fun insertPlaceholder( + context: ReceivedMessageProcessor.MessageProcessingContext, + threadId: Long, + threadAddress: Address.Conversable, + message: UnsupportedMessage, + ): MessageId? { + val sentTimestamp = message.sentTimestamp!! + val expiresInMillis = message.expiryMode.expiryMillis + val expireStartedAt = if (message.expiryMode is ExpiryMode.AfterSend) sentTimestamp else 0 + val group = threadAddress as? Address.GroupLike + + val messageId = if (message.placement == UnsupportedMessage.Placement.OUTGOING) { + if (sentTimestamp > context.maxOutgoingMessageTimestamp) { + context.maxOutgoingMessageTimestamp = sentTimestamp + } + + mmsDatabase.insertSecureDecryptedMessageOutbox( + retrieved = OutgoingMediaMessage( + recipient = threadAddress, + body = null, + attachments = emptyList(), + sentTimeMillis = sentTimestamp, + expiresInMillis = expiresInMillis, + expireStartedAtMillis = expireStartedAt, + outgoingQuote = null, + messageContent = UnsupportedMessageContent, + linkPreviews = emptyList(), + group = group, + isGroupUpdateMessage = false, + ), + threadId = threadId, + serverTimestamp = sentTimestamp, + ) + } else { + mmsDatabase.insertSecureDecryptedMessageInbox( + retrieved = IncomingMediaMessage( + from = message.sender!!.toAddress(), + sentTimeMillis = sentTimestamp, + expiresIn = expiresInMillis, + expireStartedAt = expireStartedAt, + isMessageRequestResponse = false, + hasMention = false, + body = null, + group = group, + attachments = emptyList(), + proFeatures = emptySet(), + messageContent = UnsupportedMessageContent, + quote = null, + linkPreviews = emptyList(), + dataExtractionNotification = null, + ), + threadId = threadId, + serverTimestamp = message.receivedTimestamp ?: 0, + ) + }?.let { MessageId(it.messageId, mms = true) } ?: return null + + lokiMessageDatabase.setMessageServerHash(messageId, message.serverHash!!) + + message.threadID = threadId + message.id = messageId + messageExpirationManager.get().onMessageReceived(message) + + return messageId + } + + companion object { + private const val TAG = "UnsupportedMessageHandler" + + fun currentVersion(): String = "${BuildConfig.VERSION_NAME}-${BuildConfig.VERSION_CODE}" + + /** When a retained record with no placeholder should go, given the message it holds */ + fun retainedExpiryMs(message: Message, serverTimestampMs: Long, serverExpiryMs: Long?): Long? = + UnsupportedMessageDetection.retainedExpiryMs( + afterSendStartedAtMs = message.sentTimestamp.takeIf { message.expiryMode is ExpiryMode.AfterSend }, + afterSendDurationMs = message.expiryMode.expiryMillis.takeIf { message.expiryMode is ExpiryMode.AfterSend }, + serverTimestampMs = serverTimestampMs, + serverExpiryMs = serverExpiryMs, + defaultTtlMs = SnodeMessage.DEFAULT_TTL, + ) + } +} diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageReprocessor.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageReprocessor.kt new file mode 100644 index 0000000000..c90d414498 --- /dev/null +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageReprocessor.kt @@ -0,0 +1,210 @@ +package org.session.libsession.messaging.sending_receiving + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.withContext +import network.loki.messenger.libsession_util.Namespace +import org.session.libsession.messaging.messages.Message.Companion.senderOrSync +import org.session.libsession.messaging.messages.UnsupportedMessage +import org.session.libsession.network.SnodeClock +import org.session.libsession.utilities.Address +import org.session.libsession.utilities.Address.Companion.toAddress +import org.session.libsignal.utilities.AccountId +import org.session.libsignal.utilities.Log +import org.thoughtcrime.securesms.auth.AuthAwareComponent +import org.thoughtcrime.securesms.auth.LoggedInState +import org.thoughtcrime.securesms.database.MmsDatabase +import org.thoughtcrime.securesms.database.UnsupportedMessageDatabase +import org.thoughtcrime.securesms.util.AppVisibilityManager +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.coroutines.cancellation.CancellationException + +/** + * Replays retained unsupported messages through the normal receive path the first time a new app + * version runs, so a type an update has learned replaces its placeholder at the same position (the + * replay has the same sent timestamp). Runs at launch and whenever the app comes to the foreground, + * which is also when the retention limits are enforced; after the first run for a version there is + * nothing left to attempt, so a foreground run only enforces the limits. + * + * Deliberately not a persisted job: jobs are serialised with Kryo, which can't read back anything + * holding a protobuf, and the work only needs a logged-in user, not durability. + * + * Read state carries across without copying anything, because it's decided per conversation by the + * last-seen timestamp rather than per row. For the same reason the replay doesn't notify again for a + * placeholder that was already seen. + */ +class UnsupportedMessageReprocessor @Inject constructor( + private val unsupportedMessageDatabase: UnsupportedMessageDatabase, + private val mmsDatabase: MmsDatabase, + private val messageParser: MessageParser, + private val receivedMessageProcessor: ReceivedMessageProcessor, + private val snodeClock: SnodeClock, + private val appVisibilityManager: AppVisibilityManager, +) : AuthAwareComponent { + + private enum class Result { + REPLACED, + STILL_UNSUPPORTED, + + /** This version failed to process it for another reason; the record is kept for a later version */ + FAILED, + + /** The record went before it could be reprocessed */ + MISSING, + } + + override suspend fun doWhileLoggedIn(loggedInState: LoggedInState) { + run() + + // `isAppVisible` replays its current value, which the run above already covered + appVisibilityManager.isAppVisible + .drop(1) + .filter { it } + .collect { run() } + } + + private suspend fun run() = withContext(Dispatchers.IO) { + // Neither failure may stop the other, nor end the foreground collection + try { + unsupportedMessageDatabase.enforceLimits(snodeClock.currentTimeMillis()) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to enforce retained message limits", e) + } + + try { + reprocessPending() + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to reprocess retained messages", e) + } + } + + /** + * Which records to replay is decided by the stored kind, never by the data: a group message is + * encrypted with a symmetric key, so its data can start with the byte that marks a one-to-one + * message as newer-format. + */ + private fun reprocessPending() { + val version = UnsupportedMessageHandler.currentVersion() + unsupportedMessageDatabase.stampNewerFormatAttempted(version) + + var afterId = 0L + var attempted = 0 + var replaced = 0 + var failed = 0 + + while (true) { + val ids = unsupportedMessageDatabase.unknownTypeIdsNotAttemptedBy(version, afterId, REPLAY_PAGE_SIZE) + if (ids.isEmpty()) break + + val results = receivedMessageProcessor.startProcessing("UnsupportedMessageReprocessor") { ctx -> + ids.map { id -> reprocess(ctx, id, version) } + } + + afterId = ids.last() + attempted += ids.size + replaced += results.count { it == Result.REPLACED } + failed += results.count { it == Result.FAILED } + } + + if (attempted > 0) { + Log.i(TAG, "Reprocessed $attempted retained message(s): $replaced replaced, $failed failed") + } + } + + private fun reprocess( + ctx: ReceivedMessageProcessor.MessageProcessingContext, + id: Long, + version: String, + ): Result { + val record = unsupportedMessageDatabase.get(id) ?: return Result.MISSING + val isGroup = record.namespace == Namespace.GROUP_MESSAGES() + + val result = try { + if (isGroup) { + messageParser.parseGroupMessage( + data = record.data, + serverHash = record.hash, + groupId = AccountId(record.swarmPublicKey), + currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, + currentUserId = ctx.currentUserId, + serverTimestampMs = record.serverTimestampMs, + serverExpiryMs = record.serverExpiryMs, + ) + } else { + messageParser.parse1o1Message( + data = record.data, + serverHash = record.hash, + currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, + currentUserId = ctx.currentUserId, + serverTimestampMs = record.serverTimestampMs, + serverExpiryMs = record.serverExpiryMs, + ) + } + } catch (e: Exception) { + if (e is CancellationException) throw e + return markFailed(record, version, e) + } + + if (result.message is UnsupportedMessage) { + unsupportedMessageDatabase.setLastAttemptVersion(id, version) + return Result.STILL_UNSUPPORTED + } + + val threadAddress = if (isGroup) { + Address.Group(AccountId(record.swarmPublicKey)) + } else { + result.message.senderOrSync.toAddress() as Address.Conversable + } + + try { + // The placeholder has the same sent timestamp as the replay, so it has to go first or + // the replay is discarded as a duplicate of it. It's detached first so the mms trigger + // leaves the record in place in case the replay fails. + record.placeholderMessageId?.let { placeholderId -> + unsupportedMessageDatabase.detachPlaceholder( + id = id, + expiresAtMs = record.expiresAtMs ?: UnsupportedMessageHandler.retainedExpiryMs( + message = result.message, + serverTimestampMs = record.serverTimestampMs, + serverExpiryMs = record.serverExpiryMs, + ), + ) + mmsDatabase.deleteMessage(placeholderId) + } + + receivedMessageProcessor.processSwarmMessage( + context = ctx, + threadAddress = threadAddress, + message = result.message, + proto = result.proto, + pro = result.pro, + ) + } catch (e: Exception) { + if (e is CancellationException) throw e + return markFailed(record, version, e) + } + + unsupportedMessageDatabase.delete(id) + return Result.REPLACED + } + + /** + * A failure might be specific to this version, or unrelated to the message (eg. the sender is + * now blocked), so the record is kept for a later version to retry; the retention limits still + * bound it. + */ + private fun markFailed(record: UnsupportedMessageDatabase.Record, version: String, e: Exception): Result { + Log.w(TAG, "Failed to reprocess retained message ${record.hash}: ${e.javaClass.simpleName}") + unsupportedMessageDatabase.setLastAttemptVersion(record.id, version) + return Result.FAILED + } + + companion object { + private const val TAG = "UnsupportedMessageReprocessor" + private const val REPLAY_PAGE_SIZE = 50 + } +} diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/notifications/Models.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/notifications/Models.kt index 0f7134877f..b3b4404226 100644 --- a/app/src/main/java/org/session/libsession/messaging/sending_receiving/notifications/Models.kt +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/notifications/Models.kt @@ -95,8 +95,13 @@ data class PushNotificationMetadata( @SerialName("n") val namespace: Int?, + /** When the message was stored in the swarm, in milliseconds since the epoch. */ @SerialName("t") - val timestampSeconds: Long, + val timestampMs: Long, + + /** When the message expires from the swarm, in milliseconds since the epoch. */ + @SerialName("z") + val expiryMs: Long? = null, /** The length of the message data. This is always included, even if the message content * itself was too large to fit into the push notification. */ diff --git a/app/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt b/app/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt index dbf1275f55..c2599ffac0 100644 --- a/app/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt +++ b/app/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt @@ -124,7 +124,9 @@ class Poller @Inject constructor( data = message.data, serverHash = message.hash, currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, - currentUserId = ctx.currentUserId + currentUserId = ctx.currentUserId, + serverTimestampMs = message.timestamp.toEpochMilli(), + serverExpiryMs = message.expirationMs, ) processor.processSwarmMessage( diff --git a/app/src/main/java/org/session/libsession/network/model/MessageResponses.kt b/app/src/main/java/org/session/libsession/network/model/MessageResponses.kt index 90a410afa1..9bfc63e469 100644 --- a/app/src/main/java/org/session/libsession/network/model/MessageResponses.kt +++ b/app/src/main/java/org/session/libsession/network/model/MessageResponses.kt @@ -34,6 +34,9 @@ data class RetrieveMessageResponse( @SerialName("data") val dataB64: String? = null, + + @SerialName("expiration") + val expirationMs: Long? = null, ) { val data: ByteArray by lazy { Base64.decode(dataB64, Base64.DEFAULT) diff --git a/app/src/main/java/org/thoughtcrime/securesms/auth/AuthAwareComponents.kt b/app/src/main/java/org/thoughtcrime/securesms/auth/AuthAwareComponents.kt index 32d009df3f..e4e5e36826 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/auth/AuthAwareComponents.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/auth/AuthAwareComponents.kt @@ -1,6 +1,7 @@ package org.thoughtcrime.securesms.auth import dagger.Lazy +import org.session.libsession.messaging.sending_receiving.UnsupportedMessageReprocessor import org.session.libsession.messaging.sending_receiving.pollers.PollerManager import org.thoughtcrime.securesms.attachments.AvatarUploadManager import org.thoughtcrime.securesms.configs.ConfigToDatabaseSync @@ -47,6 +48,7 @@ class AuthAwareComponents( markReadProcessor: Lazy, notificationProcessor: Lazy, versionDataFetcher: Lazy, + unsupportedMessageReprocessor: Lazy, ): this( components = listOf>( expiringMessageManager, @@ -65,6 +67,7 @@ class AuthAwareComponents( versionDataFetcher, markReadProcessor, notificationProcessor, + unsupportedMessageReprocessor, ) ) } \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationAdapter.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationAdapter.kt index 31932f96eb..f36948b089 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationAdapter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationAdapter.kt @@ -116,7 +116,7 @@ class ConversationAdapter( } ) - if (!message.isDeleted) { + if (!message.isDeleted && !message.isUnsupportedMessage) { visibleMessageView.onPress = { event -> onItemPress( message, diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationReactionOverlay.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationReactionOverlay.kt index 08d9a47ee1..eb899653e7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationReactionOverlay.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationReactionOverlay.kt @@ -635,8 +635,8 @@ class ConversationReactionOverlay : FrameLayout { val isDeprecatedLegacyGroup = recipient.isLegacyGroup && deprecationManager.isDeprecated - // control messages and "marked as deleted" messages can only delete - val isDeleteOnly = message.isDeleted || containsControlMessage + // control messages, "marked as deleted" messages and unsupported placeholders can only delete + val isDeleteOnly = message.isDeleted || containsControlMessage || message.isUnsupportedMessage // Resend if (message.isFailed && !isDeprecatedLegacyGroup) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationViewModel.kt index dc8645344d..2e09414a49 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/ConversationViewModel.kt @@ -755,7 +755,7 @@ class ConversationViewModel @AssistedInject constructor( // hashes are required if wanting to delete messages from the 'storage server' // They are not required for communities OR if all messages are outgoing // also we can only delete deleted messages and control messages (marked as deleted) locally - val canDeleteForEveryone = messages.all{ !it.isDeleted && !it.isControlMessage } && ( + val canDeleteForEveryone = messages.all{ !it.isDeleted && !it.isControlMessage && !it.isUnsupportedMessage } && ( messages.all { it.isOutgoing } || conversationType == MessageType.COMMUNITY || messages.all { lokiMessageDb.getMessageServerHash(it.messageId) != null } @@ -832,7 +832,7 @@ class ConversationViewModel @AssistedInject constructor( stopAudioIfPlaying(messages) // if the message was already marked as deleted or control messages, remove it from the db instead - if(messages.all { it.isDeleted || it.isControlMessage }){ + if(messages.all { it.isDeleted || it.isControlMessage || it.isUnsupportedMessage }){ // Remove the message locally (leave nothing behind) repository.deleteMessages(messages = messages) } else { diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/DeletedMessageView.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/DeletedMessageView.kt index 737d69de6b..3f54eadc22 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/DeletedMessageView.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/DeletedMessageView.kt @@ -23,6 +23,19 @@ class DeletedMessageView : LinearLayout { assert(message.isDeleted) // set the text to the message's body if it is set, else use a fallback binding.deleteTitleTextView.text = message.body.ifEmpty { context.resources.getQuantityString(R.plurals.deleteMessageDeleted, 1, 1) } + binding.deleteTitleTextView.contentDescription = context.getString(R.string.AccessibilityId_deleteMessageDeleted) + binding.deletedMessageViewIconImageView.setImageResource(R.drawable.ic_trash_2) + applyColor(textColor) + } + + fun bindUnsupported(@ColorInt textColor: Int) { + binding.deleteTitleTextView.setText(R.string.messageUnsupported) + binding.deleteTitleTextView.contentDescription = null + binding.deletedMessageViewIconImageView.setImageResource(R.drawable.ic_circle_alert) + applyColor(textColor) + } + + private fun applyColor(@ColorInt textColor: Int) { val deletedColor = textColor.also { alpha = 0.7f } // deleted messages use the regular text colour with some opacitiy applied) binding.deleteTitleTextView.setTextColor(deletedColor) binding.deletedMessageViewIconImageView.imageTintList = ColorStateList.valueOf(deletedColor) diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/MessageFormatter.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/MessageFormatter.kt index 9fae06aaa2..4dc0b02932 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/MessageFormatter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/MessageFormatter.kt @@ -85,6 +85,7 @@ class MessageFormatter @Inject constructor( return text } + message.isUnsupportedMessage -> return SpannableString(context.getString(R.string.messageUnsupported)) message.messageContent is DisappearingMessageUpdate -> { val isGroup = threadRecipient.isGroupOrCommunityRecipient return buildExpirationTimerMessage( @@ -184,6 +185,8 @@ class MessageFormatter @Inject constructor( ): CharSequence { return when { message.isOpenGroupInvitation -> context.getString(R.string.communityInvitation) + // Generic, like any notification that can't show the content + message.isUnsupportedMessage -> context.resources.getQuantityText(R.plurals.messageNew, 1) else -> replaceMmsAttachment(context, message, threadRecipient) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt index f0323d4323..9449177b13 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt @@ -128,10 +128,14 @@ class VisibleMessageContentView : ConstraintLayout { binding.albumThumbnailView.root.clearViews() onContentDoubleTap = null - if (message.isDeleted) { + if (message.isDeleted || message.isUnsupportedMessage) { binding.contentParent.isVisible = true binding.deletedMessageView.root.isVisible = true - binding.deletedMessageView.root.bind(message, getTextColor(context, message)) + if (message.isUnsupportedMessage) { + binding.deletedMessageView.root.bindUnsupported(getTextColor(context, message)) + } else { + binding.deletedMessageView.root.bind(message, getTextColor(context, message)) + } binding.bodyTextView.isVisible = false binding.readMore.isVisible = false binding.quoteView.root.isVisible = false diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v3/ConversationDataMapper.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v3/ConversationDataMapper.kt index 2c27b7d735..a50a6f91ea 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v3/ConversationDataMapper.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v3/ConversationDataMapper.kt @@ -173,6 +173,15 @@ class ConversationDataMapper @Inject constructor( return groups.toImmutableList() } + if (record.isUnsupportedMessage) { + addContentToGroup( + groups, + MessageContentData.Text(text = AnnotatedString(context.getString(R.string.messageUnsupported))) + ) + + return groups.toImmutableList() + } + // community invites if (record.isOpenGroupInvitation) { val jsonData = UpdateMessageData.fromJSON(json, record.body) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MmsDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MmsDatabase.kt index 66c4178652..0556fe4f4d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MmsDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MmsDatabase.kt @@ -351,6 +351,8 @@ class MmsDatabase @Inject constructor( contentValues.put(HAS_MENTION, 0) database.update(TABLE_NAME, contentValues, ID_WHERE, arrayOf(messageId.toString())) + // The row survives as a "deleted" message, so the placeholder trigger doesn't fire + database.execSQL(UnsupportedMessageDatabase.DELETE_FOR_PLACEHOLDER_SQL, arrayOf(messageId)) queue { attachmentDatabase.deleteAttachmentsForMessage(messageId) } val deletedType = if (isOutgoing) { MmsSmsColumns.Types.BASE_DELETED_OUTGOING_TYPE} else { MmsSmsColumns.Types.BASE_DELETED_INCOMING_TYPE diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabase.kt new file mode 100644 index 0000000000..99c4cbb352 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabase.kt @@ -0,0 +1,408 @@ +package org.thoughtcrime.securesms.database + +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.sqlite.db.SupportSQLiteOpenHelper +import androidx.sqlite.db.transaction +import org.session.libsession.messaging.messages.UnsupportedMessage +import org.thoughtcrime.securesms.util.asSequence +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * The raw swarm message for everything this client positively identified but can't process (see + * [UnsupportedMessage]), retained so a newer client, or a future database import, can replay it + * through its normal receive path. + * + * The table and column names are deliberately identical across the Android, iOS and Desktop + * clients so an importer can read one shape from all of them, so don't rename them. + * + * A row is removed with its placeholder by a trigger on the `mms` table rather than by each delete + * path, so no current or future way of deleting messages or conversations can leave rows behind. + */ +@Singleton +class UnsupportedMessageDatabase @Inject constructor( + private val helper: Provider, +) { + private val readableDatabase: SupportSQLiteDatabase get() = helper.get().readableDatabase + private val writableDatabase: SupportSQLiteDatabase get() = helper.get().writableDatabase + + class Record( + val id: Long, + val kind: UnsupportedMessage.Kind, + val swarmPublicKey: String, + val namespace: Int, + val hash: String, + val sender: String?, + val sentTimestampMs: Long?, + val serverTimestampMs: Long, + val serverExpiryMs: Long?, + val data: ByteArray, + val placeholderMessageId: Long?, + val expiresAtMs: Long?, + val receivedAtMs: Long, + val lastAttemptVersion: String, + ) + + fun exists(hash: String): Boolean { + //language=roomsql + return readableDatabase.query("SELECT 1 FROM $TABLE_NAME WHERE hash = ?", arrayOf(hash)) + .use { it.moveToNext() } + } + + /** + * @return false if a record with the same hash already exists + */ + fun insert( + kind: UnsupportedMessage.Kind, + swarmPublicKey: String, + namespace: Int, + hash: String, + sender: String?, + sentTimestampMs: Long?, + serverTimestampMs: Long, + serverExpiryMs: Long?, + data: ByteArray, + placeholderMessageId: Long?, + expiresAtMs: Long?, + receivedAtMs: Long, + lastAttemptVersion: String, + ): Boolean { + //language=roomsql + return writableDatabase.compileStatement(""" + INSERT OR IGNORE INTO $TABLE_NAME ( + kind, swarm_public_key, namespace, hash, sender, sent_timestamp_ms, server_timestamp_ms, + server_expiry_ms, data, placeholder_message_id, expires_at_ms, received_at_ms, + last_attempt_version + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """).use { stmt -> + stmt.bindString(1, kind.dbValue) + stmt.bindString(2, swarmPublicKey) + stmt.bindLong(3, namespace.toLong()) + stmt.bindString(4, hash) + sender?.let { stmt.bindString(5, it) } ?: stmt.bindNull(5) + sentTimestampMs?.let { stmt.bindLong(6, it) } ?: stmt.bindNull(6) + stmt.bindLong(7, serverTimestampMs) + serverExpiryMs?.let { stmt.bindLong(8, it) } ?: stmt.bindNull(8) + stmt.bindBlob(9, data) + placeholderMessageId?.let { stmt.bindLong(10, it) } ?: stmt.bindNull(10) + expiresAtMs?.let { stmt.bindLong(11, it) } ?: stmt.bindNull(11) + stmt.bindLong(12, receivedAtMs) + stmt.bindString(13, lastAttemptVersion) + stmt.executeInsert() != -1L + } + } + + fun get(id: Long): Record? { + //language=roomsql + return readableDatabase.query("SELECT * FROM $TABLE_NAME WHERE id = ?", arrayOf(id)).use { cursor -> + if (!cursor.moveToNext()) return@use null + + val kind = UnsupportedMessage.Kind.fromDbValue(cursor.getString(cursor.getColumnIndexOrThrow("kind"))) + ?: return@use null + + fun nullableLong(column: String): Long? = cursor.getColumnIndexOrThrow(column) + .takeUnless(cursor::isNull) + ?.let(cursor::getLong) + + Record( + id = cursor.getLong(cursor.getColumnIndexOrThrow("id")), + kind = kind, + swarmPublicKey = cursor.getString(cursor.getColumnIndexOrThrow("swarm_public_key")), + namespace = cursor.getInt(cursor.getColumnIndexOrThrow("namespace")), + hash = cursor.getString(cursor.getColumnIndexOrThrow("hash")), + sender = cursor.getColumnIndexOrThrow("sender").takeUnless(cursor::isNull)?.let(cursor::getString), + sentTimestampMs = nullableLong("sent_timestamp_ms"), + serverTimestampMs = cursor.getLong(cursor.getColumnIndexOrThrow("server_timestamp_ms")), + serverExpiryMs = nullableLong("server_expiry_ms"), + data = cursor.getBlob(cursor.getColumnIndexOrThrow("data")), + placeholderMessageId = nullableLong("placeholder_message_id"), + expiresAtMs = nullableLong("expires_at_ms"), + receivedAtMs = cursor.getLong(cursor.getColumnIndexOrThrow("received_at_ms")), + lastAttemptVersion = cursor.getString(cursor.getColumnIndexOrThrow("last_attempt_version")), + ) + } + } + + /** + * Marks every newer-format record as attempted by [version] without loading any of them: no + * version of this client can decrypt one, so replaying it could only end where it started. + */ + fun stampNewerFormatAttempted(version: String): Int { + //language=roomsql + return writableDatabase.compileStatement( + "UPDATE $TABLE_NAME SET last_attempt_version = ? WHERE kind = ? AND last_attempt_version != ?" + ).use { stmt -> + stmt.bindString(1, version) + stmt.bindString(2, UnsupportedMessage.Kind.NEWER_FORMAT.dbValue) + stmt.bindString(3, version) + stmt.executeUpdateDelete() + } + } + + /** + * A page of unknown-type ids not yet attempted by [version], after [afterId]. Ids only, so the + * retained data is never loaded more than a record at a time. + */ + fun unknownTypeIdsNotAttemptedBy(version: String, afterId: Long, limit: Int): List { + //language=roomsql + return readableDatabase.query( + """ + SELECT id FROM $TABLE_NAME + WHERE kind = ? AND last_attempt_version != ? AND id > ? + ORDER BY id + LIMIT ? + """, + arrayOf(UnsupportedMessage.Kind.UNKNOWN_TYPE.dbValue, version, afterId, limit) + ).use { cursor -> cursor.asSequence().map { it.getLong(0) }.toList() } + } + + fun setLastAttemptVersion(id: Long, version: String) { + //language=roomsql + writableDatabase.execSQL( + "UPDATE $TABLE_NAME SET last_attempt_version = ? WHERE id = ?", + arrayOf(version, id) + ) + } + + fun delete(id: Long) { + //language=roomsql + writableDatabase.execSQL("DELETE FROM $TABLE_NAME WHERE id = ?", arrayOf(id)) + } + + /** + * Removes the records an unsend request targets. A record without a placeholder can't be found + * through a message row, and replaying it after an update would restore a message its sender + * deleted. + */ + fun deleteForUnsend(sender: String, sentTimestampMs: Long) { + //language=roomsql + writableDatabase.execSQL( + "DELETE FROM $TABLE_NAME WHERE sender = ? AND sent_timestamp_ms = ?", + arrayOf(sender, sentTimestampMs) + ) + } + + /** + * Unlinks the record from its placeholder so the placeholder can be deleted without the trigger + * taking the record with it, and gives the record its own [expiresAtMs] since nothing else owns + * its expiry any more. + */ + fun detachPlaceholder(id: Long, expiresAtMs: Long?) { + //language=roomsql + writableDatabase.execSQL( + "UPDATE $TABLE_NAME SET placeholder_message_id = NULL, expires_at_ms = ? WHERE id = ?", + arrayOf(expiresAtMs, id) + ) + } + + /** + * Removes rows whose message has expired, then applies [enforceRetentionCaps]. + */ + fun enforceLimits( + nowMs: Long, + maxRetainedBytes: Long = MAX_RETAINED_BYTES, + maxNewerFormatCount: Long = MAX_NEWER_FORMAT_COUNT, + ) { + writableDatabase.transaction { + //language=roomsql + execSQL("DELETE FROM $TABLE_NAME WHERE expires_at_ms <= ?", arrayOf(nowMs)) + applyRetentionCaps(maxRetainedBytes, maxNewerFormatCount) + } + } + + /** + * Cheap enough to run after every insert: it reads the single [STATS_TABLE_NAME] row, and only + * touches `unsupported_message` (through `unsupported_message_kind_id`) when a cap is exceeded. + */ + fun enforceRetentionCaps( + maxRetainedBytes: Long = MAX_RETAINED_BYTES, + maxNewerFormatCount: Long = MAX_NEWER_FORMAT_COUNT, + ) { + writableDatabase.transaction { applyRetentionCaps(maxRetainedBytes, maxNewerFormatCount) } + } + + /** + * Newer-format rows go before unknown-type rows: anyone can deposit data with the newer-format + * prefix in a one-to-one namespace, whereas an unknown type came from an authenticated sender. + * Evicting an unknown type only drops its bytes; its placeholder stays, permanently unreadable. + * + * Every count and total here comes from [STATS_TABLE_NAME], which is only correct while its + * triggers see every change to `unsupported_message`. + */ + private fun SupportSQLiteDatabase.applyRetentionCaps(maxRetainedBytes: Long, maxNewerFormatCount: Long) { + val excessNewerFormat = readStats().newerFormatCount - maxNewerFormatCount + if (excessNewerFormat > 0) { + //language=roomsql + execSQL( + """ + DELETE FROM $TABLE_NAME WHERE id IN ( + SELECT id FROM $TABLE_NAME WHERE kind = ? ORDER BY id ASC LIMIT ? + ) + """, + arrayOf(UnsupportedMessage.Kind.NEWER_FORMAT.dbValue, excessNewerFormat) + ) + } + + while (true) { + var excessBytes = readStats().totalBytes - maxRetainedBytes + if (excessBytes <= 0) return + + val batch = mutableListOf() + + for (kind in EVICTION_ORDER) { + //language=roomsql + query( + "SELECT id, length(data) FROM $TABLE_NAME WHERE kind = ? ORDER BY id ASC LIMIT ?", + arrayOf(kind.dbValue, EVICTION_BATCH_SIZE) + ).use { cursor -> + while (excessBytes > 0 && cursor.moveToNext()) { + batch += cursor.getLong(0) + excessBytes -= cursor.getLong(1) + ROW_OVERHEAD_BYTES + } + } + + if (batch.isNotEmpty()) break + } + + // Only reachable if the stats row disagrees with the table, in which case looping + // would never end + if (batch.isEmpty()) return + + //language=roomsql + execSQL( + "DELETE FROM $TABLE_NAME WHERE id IN (SELECT value FROM json_each(?))", + arrayOf(batch.joinToString(prefix = "[", postfix = "]")) + ) + } + } + + private class Stats(val totalBytes: Long, val newerFormatCount: Long) + + private fun SupportSQLiteDatabase.readStats(): Stats { + //language=roomsql + return query("SELECT total_bytes, newer_format_count FROM $STATS_TABLE_NAME WHERE id = 1").use { + if (it.moveToNext()) Stats(it.getLong(0), it.getLong(1)) else Stats(0, 0) + } + } + + companion object { + const val TABLE_NAME = "unsupported_message" + + const val STATS_TABLE_NAME = "unsupported_message_stats" + + /** Counted as `length(data) + ROW_OVERHEAD_BYTES` per row, so tiny rows still cost something */ + const val MAX_RETAINED_BYTES: Long = 256L * 1024 * 1024 + + /** + * Written into the stats triggers when the table is created, so changing it needs a migration which + * recreates those triggers and recomputes `total_bytes`, otherwise existing databases keep counting + * the old value. + */ + const val ROW_OVERHEAD_BYTES: Long = 256 + const val MAX_NEWER_FORMAT_COUNT: Long = 10_000 + + private const val EVICTION_BATCH_SIZE = 100 + private val EVICTION_ORDER = listOf(UnsupportedMessage.Kind.NEWER_FORMAT, UnsupportedMessage.Kind.UNKNOWN_TYPE) + + //language=roomsql + const val DELETE_FOR_PLACEHOLDER_SQL = "DELETE FROM $TABLE_NAME WHERE placeholder_message_id = ?" + + @JvmStatic + fun createTable(db: SupportSQLiteDatabase) { + //language=roomsql + db.execSQL(""" + CREATE TABLE IF NOT EXISTS $TABLE_NAME ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + swarm_public_key TEXT NOT NULL, + namespace INTEGER NOT NULL, + hash TEXT NOT NULL UNIQUE, + sender TEXT, + sent_timestamp_ms INTEGER, + server_timestamp_ms INTEGER NOT NULL, + server_expiry_ms INTEGER, + data BLOB NOT NULL, + placeholder_message_id INTEGER, + expires_at_ms INTEGER, + received_at_ms INTEGER NOT NULL, + last_attempt_version TEXT NOT NULL + ) + """) + + //language=roomsql + db.execSQL("CREATE INDEX IF NOT EXISTS unsupported_message_placeholder_index ON $TABLE_NAME (placeholder_message_id)") + + //language=roomsql + db.execSQL("CREATE INDEX IF NOT EXISTS unsupported_message_expires_at_index ON $TABLE_NAME (expires_at_ms)") + + //language=roomsql + db.execSQL("CREATE INDEX IF NOT EXISTS unsupported_message_sender_index ON $TABLE_NAME (sender, sent_timestamp_ms)") + + //language=roomsql + db.execSQL("CREATE INDEX IF NOT EXISTS unsupported_message_kind_id ON $TABLE_NAME (kind, id)") + + //language=roomsql + db.execSQL(""" + CREATE TABLE IF NOT EXISTS $STATS_TABLE_NAME ( + id INTEGER PRIMARY KEY CHECK (id = 1), + total_bytes INTEGER NOT NULL DEFAULT 0, + newer_format_count INTEGER NOT NULL DEFAULT 0 + ) + """) + + //language=roomsql + db.execSQL("INSERT OR IGNORE INTO $STATS_TABLE_NAME (id, total_bytes, newer_format_count) VALUES (1, 0, 0)") + + val newerFormat = "'${UnsupportedMessage.Kind.NEWER_FORMAT.dbValue}'" + + //language=roomsql + db.execSQL(""" + CREATE TRIGGER IF NOT EXISTS unsupported_message_stats_insert + AFTER INSERT ON $TABLE_NAME + BEGIN + UPDATE $STATS_TABLE_NAME SET + total_bytes = total_bytes + length(NEW.data) + $ROW_OVERHEAD_BYTES, + newer_format_count = newer_format_count + (NEW.kind = $newerFormat) + WHERE id = 1; + END + """) + + //language=roomsql + db.execSQL(""" + CREATE TRIGGER IF NOT EXISTS unsupported_message_stats_delete + AFTER DELETE ON $TABLE_NAME + BEGIN + UPDATE $STATS_TABLE_NAME SET + total_bytes = total_bytes - length(OLD.data) - $ROW_OVERHEAD_BYTES, + newer_format_count = newer_format_count - (OLD.kind = $newerFormat) + WHERE id = 1; + END + """) + + //language=roomsql + db.execSQL(""" + CREATE TRIGGER IF NOT EXISTS unsupported_message_stats_update + AFTER UPDATE OF data, kind ON $TABLE_NAME + BEGIN + UPDATE $STATS_TABLE_NAME SET + total_bytes = total_bytes - length(OLD.data) + length(NEW.data), + newer_format_count = newer_format_count - (OLD.kind = $newerFormat) + (NEW.kind = $newerFormat) + WHERE id = 1; + END + """) + + // Placeholders are only ever MMS rows: SMS ids overlap with MMS ids, so there must be + // no equivalent on the sms table. + //language=roomsql + db.execSQL(""" + CREATE TRIGGER IF NOT EXISTS unsupported_message_placeholder_delete + AFTER DELETE ON ${MmsDatabase.TABLE_NAME} + BEGIN + DELETE FROM $TABLE_NAME WHERE placeholder_message_id = OLD.${MmsSmsColumns.ID}; + END + """) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 91aff24375..f1c493149e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -42,6 +42,7 @@ import org.thoughtcrime.securesms.database.SmsDatabase; import org.thoughtcrime.securesms.database.SnodeDatabase; import org.thoughtcrime.securesms.database.ThreadDatabase; +import org.thoughtcrime.securesms.database.UnsupportedMessageDatabase; import org.thoughtcrime.securesms.pro.db.ProDatabase; import org.thoughtcrime.securesms.util.ConfigurationMessageUtilities; @@ -113,9 +114,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int lokiV59 = 80; private static final int lokiV60 = 81; private static final int lokiV61 = 82; + private static final int lokiV62 = 83; // Loki - onUpgrade(...) must be updated to use Loki version numbers if Signal makes any database changes - private static final int DATABASE_VERSION = lokiV61; + private static final int DATABASE_VERSION = lokiV62; private static final int MIN_DATABASE_VERSION = lokiV7; public static final String DATABASE_NAME = "session.db"; @@ -312,6 +314,8 @@ public void onCreate(SQLiteDatabase db) { MmsDatabase.Companion.addOutgoingColumn(db); ProDatabase.Companion.reshapeRevocationsForSeconds(db); + + UnsupportedMessageDatabase.createTable(db); } @Override @@ -679,6 +683,10 @@ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { ProDatabase.Companion.reshapeRevocationsForSeconds(db); } + if (oldVersion < lokiV62) { + UnsupportedMessageDatabase.createTable(db); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/model/DisplayRecord.java b/app/src/main/java/org/thoughtcrime/securesms/database/model/DisplayRecord.java index c141c5e7b3..9e3ed088c0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/model/DisplayRecord.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/model/DisplayRecord.java @@ -23,6 +23,7 @@ import org.thoughtcrime.securesms.database.MmsSmsColumns; import org.thoughtcrime.securesms.database.SmsDatabase; import org.thoughtcrime.securesms.database.model.content.DisappearingMessageUpdate; +import org.thoughtcrime.securesms.database.model.content.UnsupportedMessageContent; import org.thoughtcrime.securesms.database.model.content.MessageContent; /** @@ -123,5 +124,12 @@ public boolean isControlMessage() { (messageContent instanceof DisappearingMessageUpdate); } + /** + * A placeholder for a message this client can't display, until it's deleted. + */ + public boolean isUnsupportedMessage() { + return !isDeleted() && messageContent instanceof UnsupportedMessageContent; + } + public boolean isGroupV2ExpirationTimerUpdate() { return false; } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/model/content/MessageContentModule.kt b/app/src/main/java/org/thoughtcrime/securesms/database/model/content/MessageContentModule.kt index bd8e3cdb76..b490addab0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/model/content/MessageContentModule.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/model/content/MessageContentModule.kt @@ -18,6 +18,7 @@ class MessageContentModule { return SerializersModule { polymorphic(MessageContent::class) { subclass(DisappearingMessageUpdate::class) + subclass(UnsupportedMessageContent::class) defaultDeserializer { UnknownMessageContent.serializer() } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContent.kt b/app/src/main/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContent.kt new file mode 100644 index 0000000000..974bd6f044 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContent.kt @@ -0,0 +1,14 @@ +package org.thoughtcrime.securesms.database.model.content + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * A placeholder for a message whose type this client doesn't know. The raw message is retained in + * `unsupported_message`, which points back at this row and is replayed once an update knows the type. + */ +@Serializable +@SerialName(UnsupportedMessageContent.TYPE_NAME) +data object UnsupportedMessageContent : MessageContent { + const val TYPE_NAME = "unsupported_message" +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/groups/GroupPoller.kt b/app/src/main/java/org/thoughtcrime/securesms/groups/GroupPoller.kt index 4196267a8c..c2de9e6804 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/groups/GroupPoller.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/groups/GroupPoller.kt @@ -332,6 +332,8 @@ class GroupPoller @AssistedInject constructor( groupId = groupId, currentUserId = ctx.currentUserId, currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, + serverTimestampMs = message.timestamp.toEpochMilli(), + serverExpiryMs = message.expirationMs, ) receivedMessageProcessor.processSwarmMessage( diff --git a/app/src/main/java/org/thoughtcrime/securesms/home/HomeActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/home/HomeActivity.kt index 5566aba642..cbaf448e88 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/home/HomeActivity.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/home/HomeActivity.kt @@ -53,6 +53,7 @@ import network.loki.messenger.libsession_util.PRIORITY_HIDDEN import org.session.libsession.messaging.groups.GroupManagerV2 import org.session.libsession.messaging.groups.LegacyGroupDeprecationManager import org.session.libsession.messaging.jobs.JobQueue +import org.session.libsession.messaging.sending_receiving.UnsupportedMessageBannerStore import org.session.libsession.network.SnodeClock import org.session.libsession.network.model.PathStatus import org.session.libsession.network.onion.PathManager @@ -162,6 +163,7 @@ class HomeActivity : ScreenLockActionBarActivity(), @Inject lateinit var contentViewFactory: GlobalSearchAdapter.ContentView.Factory @Inject lateinit var jobQueue: Provider @Inject lateinit var snodeDirectory: SnodeDirectory + @Inject lateinit var unsupportedMessageBannerStore: UnsupportedMessageBannerStore private val globalSearchViewModel by viewModels() private val homeViewModel by viewModels() @@ -393,6 +395,19 @@ class HomeActivity : ScreenLockActionBarActivity(), updateLegacyConfigView() } + binding.unsupportedMessageBannerDismiss.setOnClickListener { + unsupportedMessageBannerStore.dismiss(clock.currentTimeMillis()) + } + + lifecycleScope.launch { + repeatOnLifecycle(Lifecycle.State.STARTED) { + unsupportedMessageBannerStore.state.collect { state -> + binding.unsupportedMessageBanner.isVisible = state.textRes != null + state.textRes?.let { binding.unsupportedMessageBannerTextView.setText(it) } + } + } + } + // in case a phone call is in progress, this banner is visible and should bring the user back to the call binding.callInProgress.setOnClickListener { startActivity(WebRtcCallActivity.getCallActivityIntent(this)) diff --git a/app/src/main/java/org/thoughtcrime/securesms/notifications/PushReceiver.kt b/app/src/main/java/org/thoughtcrime/securesms/notifications/PushReceiver.kt index ae4e0749b6..f1e5ade23c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/notifications/PushReceiver.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/notifications/PushReceiver.kt @@ -108,7 +108,9 @@ class PushReceiver @Inject constructor( serverHash = pushData.metadata.msg_hash, groupId = groupId, currentUserId = ctx.currentUserId, - currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data + currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, + serverTimestampMs = pushData.metadata.timestampMs, + serverExpiryMs = pushData.metadata.expiryMs, ) receivedMessageProcessor.processSwarmMessage( @@ -139,7 +141,7 @@ class PushReceiver @Inject constructor( ConfigMessage( hash = hash, data = pushData.data, - timestamp = pushData.metadata.timestampSeconds + timestamp = pushData.metadata.timestampMs ) ) @@ -179,6 +181,10 @@ class PushReceiver @Inject constructor( serverHash = pushData.metadata?.msg_hash, currentUserId = ctx.currentUserId, currentUserEd25519PrivKey = ctx.currentUserEd25519KeyPair.secretKey.data, + // Without metadata there's no hash either, and the swarm + // origin is only retained for a message with one. + serverTimestampMs = pushData.metadata?.timestampMs ?: 0L, + serverExpiryMs = pushData.metadata?.expiryMs, ) receivedMessageProcessor.processSwarmMessage( diff --git a/app/src/main/res/drawable/ic_circle_alert.xml b/app/src/main/res/drawable/ic_circle_alert.xml new file mode 100644 index 0000000000..cfd83e9d33 --- /dev/null +++ b/app/src/main/res/drawable/ic_circle_alert.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/app/src/main/res/layout/activity_home.xml b/app/src/main/res/layout/activity_home.xml index 910c965caf..c69d59122c 100644 --- a/app/src/main/res/layout/activity_home.xml +++ b/app/src/main/res/layout/activity_home.xml @@ -144,6 +144,38 @@ app:tint="@color/black" /> + + + + + () + + private lateinit var parser: MessageParser + + @Before + fun setUp() { + Log.initialize(NoOpLogger) + + val userConfigs = mockk { + every { contacts.get(any()) } answers { + val id = firstArg() + if (id in blockedContacts) mockk { every { blocked } returns true } else null + } + } + + parser = MessageParser( + configFactory = mockk { + every { dangerouslyAccessUserConfigs() } returns (userConfigs to {}) + }, + storage = mockk(relaxed = true), + snodeClock = mockk { every { currentTimeMillis() } returns messageTimestampMs + 1 }, + proBackendConfig = { mockk() }, + proDatabase = mockk(relaxed = true), + ) + } + + private fun content(build: SessionProtos.Content.Builder.() -> Unit): SessionProtos.Content = + SessionProtos.Content.newBuilder().apply(build).build() + + private fun hasValidKnownContent(content: SessionProtos.Content, isForGroup: Boolean = false) = + parser.hasValidKnownContent( + proto = content, + isForGroup = isForGroup, + sender = otherUser, + currentUserId = currentUser, + messageTimestampMs = messageTimestampMs, + ) + + private fun isUnknownType(content: SessionProtos.Content): Boolean { + val plaintext = content.toByteArray() + unknownField + return UnsupportedMessageDetection.isUnknownType(plaintext) { + hasValidKnownContent(SessionProtos.Content.parseFrom(plaintext)) + } + } + + private fun parseUnknown( + content: SessionProtos.Content, + sender: AccountId, + isForGroup: Boolean = false, + ): UnsupportedMessage = parser.parseUnknownTypeMessage( + proto = content, + sender = sender, + messageTimestampMs = messageTimestampMs, + checkForBlockStatus = true, + isForGroup = isForGroup, + currentUserId = currentUser, + swarmOrigin = MessageParser.SwarmOrigin( + rawData = byteArrayOf(1, 2, 3), + swarmPublicKey = currentUser.hexString, + namespace = 0, + serverHash = "synthetic-hash", + serverTimestampMs = messageTimestampMs, + serverExpiryMs = null, + ), + ).message as UnsupportedMessage + + @Test + fun `known content with an unknown field is processed normally`() { + assertFalse(isUnknownType(content { + setDataMessage(SessionProtos.DataMessage.newBuilder().setBody("synthetic")) + })) + } + + @Test + fun `an unknown field with no known content is an unknown type`() { + assertTrue(isUnknownType(content { setSigTimestamp(messageTimestampMs) })) + } + + @Test + fun `a sync copy carrying only a sync target is an unknown type`() { + assertTrue(isUnknownType(content { + setDataMessage(SessionProtos.DataMessage.newBuilder().setSyncTarget(syncTarget)) + })) + } + + @Test + fun `invalid known content without an unknown field is not an unknown type`() { + val plaintext = content { setDataMessage(SessionProtos.DataMessage.newBuilder()) }.toByteArray() + + assertFalse(UnsupportedMessageDetection.isUnknownType(plaintext) { + hasValidKnownContent(SessionProtos.Content.parseFrom(plaintext)) + }) + } + + @Test + fun `an unknown type from someone else is incoming and keeps its expiry`() { + val message = parseUnknown( + content { + expirationType = SessionProtos.Content.ExpirationType.DELETE_AFTER_SEND + expirationTimer = 60 + }, + sender = otherUser, + ) + + assertEquals(UnsupportedMessage.Kind.UNKNOWN_TYPE, message.kind) + assertEquals(UnsupportedMessage.Placement.INCOMING, message.placement) + assertEquals(otherUser.hexString, message.sender) + assertEquals(messageTimestampMs, message.sentTimestamp) + assertEquals("synthetic-hash", message.serverHash) + assertEquals(ExpiryMode.AfterSend(60), message.expiryMode) + } + + @Test + fun `an unknown type from a blocked sender is dropped`() { + blockedContacts += otherUser.hexString + + assertThrows(NonRetryableException::class.java) { + parseUnknown(content { }, sender = otherUser) + } + } + + @Test + fun `a sync from our own device is placed in its sync target`() { + val message = parseUnknown( + content { setDataMessage(SessionProtos.DataMessage.newBuilder().setSyncTarget(syncTarget)) }, + sender = currentUser, + ) + + assertEquals(UnsupportedMessage.Placement.OUTGOING, message.placement) + assertEquals(syncTarget, message.syncTarget) + } + + @Test + fun `a sync from our own device without a sync target has no placement`() { + val message = parseUnknown(content { }, sender = currentUser) + + assertEquals(UnsupportedMessage.Placement.NONE, message.placement) + assertNull(message.syncTarget) + } + + @Test + fun `a group message from our own device is outgoing in the group`() { + val message = parseUnknown(content { }, sender = currentUser, isForGroup = true) + + assertEquals(UnsupportedMessage.Placement.OUTGOING, message.placement) + } +} diff --git a/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBannerTest.kt b/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBannerTest.kt new file mode 100644 index 0000000000..15bb89df59 --- /dev/null +++ b/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageBannerTest.kt @@ -0,0 +1,51 @@ +package org.session.libsession.messaging.sending_receiving + +import org.junit.Assert.assertEquals +import org.junit.Test +import org.session.libsession.messaging.sending_receiving.UnsupportedMessageBanner.REAPPEAR_AFTER_DISMISSAL_MS +import org.session.libsession.messaging.sending_receiving.UnsupportedMessageBanner.State + +class UnsupportedMessageBannerTest { + + private fun state(triggered: Long?, otherDevice: Long? = null, dismissed: Long? = null) = + UnsupportedMessageBanner.state(triggeredAtMs = triggered, otherDeviceTriggeredAtMs = otherDevice, dismissedAtMs = dismissed) + + @Test + fun `hidden until triggered`() { + assertEquals(State.HIDDEN, state(triggered = null)) + assertEquals(State.HIDDEN, state(triggered = null, dismissed = 1_000L)) + } + + @Test + fun `general text when only a newer format triggered it and it was never dismissed`() { + // A missing threshold must not make a missing other-device trigger count as current + assertEquals(State.GENERAL, state(triggered = 1_000L)) + } + + @Test + fun `other device text wins when that was a trigger`() { + assertEquals(State.OTHER_DEVICE, state(triggered = 2_000L, otherDevice = 1_000L)) + } + + @Test + fun `only reappears for a trigger at least a week after dismissal`() { + val dismissed = 10_000L + + assertEquals(State.HIDDEN, state(triggered = dismissed + REAPPEAR_AFTER_DISMISSAL_MS - 1, dismissed = dismissed)) + assertEquals(State.GENERAL, state(triggered = dismissed + REAPPEAR_AFTER_DISMISSAL_MS, dismissed = dismissed)) + } + + @Test + fun `an other device trigger from before the dismissal no longer picks its text`() { + val dismissed = 10_000L + val triggered = dismissed + REAPPEAR_AFTER_DISMISSAL_MS + + assertEquals(State.GENERAL, state(triggered = triggered, otherDevice = dismissed - 1, dismissed = dismissed)) + assertEquals(State.OTHER_DEVICE, state(triggered = triggered, otherDevice = triggered, dismissed = dismissed)) + } + + @Test + fun `a dismissal after the last trigger hides it`() { + assertEquals(State.HIDDEN, state(triggered = 1_000L, otherDevice = 1_000L, dismissed = 2_000L)) + } +} diff --git a/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetectionTest.kt b/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetectionTest.kt new file mode 100644 index 0000000000..9d00125cae --- /dev/null +++ b/app/src/test/java/org/session/libsession/messaging/sending_receiving/UnsupportedMessageDetectionTest.kt @@ -0,0 +1,151 @@ +package org.session.libsession.messaging.sending_receiving + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.session.libsession.messaging.messages.UnsupportedMessage.Placement +import org.session.protos.SessionProtos + +/** + * Synthetic payloads only: every byte here is built by the test. + */ +class UnsupportedMessageDetectionTest { + + private val knownText: ByteArray = SessionProtos.Content.newBuilder() + .setDataMessage(SessionProtos.DataMessage.newBuilder().setBody("synthetic")) + .build() + .toByteArray() + + // Field 18 (msgId in newer clients), varint 1 + private val field18 = byteArrayOf(0x90.toByte(), 0x01, 0x01) + + // Field 19, length-delimited, two bytes + private val field19 = byteArrayOf(0x9A.toByte(), 0x01, 0x02, 0xAA.toByte(), 0xBB.toByte()) + + @Test + fun `scan returns the top-level field numbers in order`() { + val data = knownText + field18 + field19 + + // DataMessage is field 1 of Content + assertEquals(listOf(1, 18, 19), UnsupportedMessageDetection.topLevelFieldNumbers(data)) + } + + @Test + fun `scan handles every supported wire type`() { + val data = byteArrayOf( + 0x08, 0x96.toByte(), 0x01, // 1: varint 150 + 0x11, 1, 2, 3, 4, 5, 6, 7, 8, // 2: fixed64 + 0x1A, 0x01, 0x00, // 3: length-delimited, 1 byte + 0x25, 1, 2, 3, 4, // 4: fixed32 + ) + + assertEquals(listOf(1, 2, 3, 4), UnsupportedMessageDetection.topLevelFieldNumbers(data)) + } + + @Test + fun `scan of empty data is an empty list`() { + assertEquals(emptyList(), UnsupportedMessageDetection.topLevelFieldNumbers(ByteArray(0))) + } + + @Test + fun `scan rejects malformed data`() { + // Truncated length-delimited field + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(0x9A.toByte(), 0x01, 0x05, 0x00))) + // Truncated fixed64 / fixed32 + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(0x11, 1, 2, 3))) + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(0x25, 1, 2))) + // Unterminated varint + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(0x08, 0x80.toByte()))) + // Field number 0 + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(0x00, 0x01))) + // Group wire types (3, 4) and the undefined 6, 7 + for (wireType in listOf(3, 4, 6, 7)) { + assertNull(UnsupportedMessageDetection.topLevelFieldNumbers(byteArrayOf(((1 shl 3) or wireType).toByte(), 0x00))) + } + } + + @Test + fun `fields up to 18 are known`() { + assertFalse(UnsupportedMessageDetection.containsUnknownContentType(knownText)) + assertFalse(UnsupportedMessageDetection.containsUnknownContentType(knownText + field18)) + assertTrue(UnsupportedMessageDetection.containsUnknownContentType(knownText + field19)) + } + + @Test + fun `malformed content is never an unknown type`() { + assertFalse(UnsupportedMessageDetection.containsUnknownContentType(field19.copyOf(3))) + } + + @Test + fun `an unknown field only counts without valid known content`() { + val unknown = knownText + field19 + + assertTrue(UnsupportedMessageDetection.isUnknownType(unknown) { false }) + assertFalse(UnsupportedMessageDetection.isUnknownType(unknown) { true }) + assertFalse(UnsupportedMessageDetection.isUnknownType(knownText + field18) { false }) + assertFalse(UnsupportedMessageDetection.isUnknownType(ByteArray(0)) { false }) + } + + @Test + fun `content with an unknown field still parses with the generated classes`() { + val content = SessionProtos.Content.parseFrom(knownText + field19) + + assertEquals("synthetic", content.dataMessage.body) + } + + @Test + fun `only data starting with a zero byte is a newer format`() { + assertTrue(UnsupportedMessageDetection.isNewerFormat(byteArrayOf(0x00, 0x02, 0x7F))) + assertTrue(UnsupportedMessageDetection.isNewerFormat(byteArrayOf(0x00, 0x05))) + assertFalse(UnsupportedMessageDetection.isNewerFormat(knownText)) + assertFalse(UnsupportedMessageDetection.isNewerFormat(ByteArray(0))) + } + + @Test + fun `placement follows the sender and sync target`() { + val target = "05" + "ab".repeat(32) + + assertEquals(Placement.INCOMING to null, UnsupportedMessageDetection.placement(isGroup = false, isSenderSelf = false, syncTarget = target)) + assertEquals(Placement.OUTGOING to target, UnsupportedMessageDetection.placement(isGroup = false, isSenderSelf = true, syncTarget = target)) + assertEquals(Placement.NONE to null, UnsupportedMessageDetection.placement(isGroup = false, isSenderSelf = true, syncTarget = null)) + // Only a standard (05) account can be a sync target + assertEquals(Placement.NONE to null, UnsupportedMessageDetection.placement(isGroup = false, isSenderSelf = true, syncTarget = "03" + "ab".repeat(32))) + assertEquals(Placement.NONE to null, UnsupportedMessageDetection.placement(isGroup = false, isSenderSelf = true, syncTarget = "not an id")) + assertEquals(Placement.INCOMING to null, UnsupportedMessageDetection.placement(isGroup = true, isSenderSelf = false, syncTarget = null)) + assertEquals(Placement.OUTGOING to null, UnsupportedMessageDetection.placement(isGroup = true, isSenderSelf = true, syncTarget = target)) + } + + @Test + fun `retained expiry prefers an after-send timer`() { + assertEquals( + 1_000L + 60_000L, + UnsupportedMessageDetection.retainedExpiryMs( + afterSendStartedAtMs = 1_000L, + afterSendDurationMs = 60_000L, + serverTimestampMs = 900L, + serverExpiryMs = 900L + DEFAULT_TTL, + defaultTtlMs = DEFAULT_TTL, + ) + ) + } + + @Test + fun `retained expiry uses the swarm expiry only when shorter than the default TTL`() { + assertEquals( + 5_000L, + UnsupportedMessageDetection.retainedExpiryMs(null, null, serverTimestampMs = 1_000L, serverExpiryMs = 5_000L, defaultTtlMs = DEFAULT_TTL) + ) + assertNull( + UnsupportedMessageDetection.retainedExpiryMs(null, null, serverTimestampMs = 1_000L, serverExpiryMs = 1_000L + DEFAULT_TTL, defaultTtlMs = DEFAULT_TTL) + ) + assertNull( + UnsupportedMessageDetection.retainedExpiryMs(null, null, serverTimestampMs = 1_000L, serverExpiryMs = null, defaultTtlMs = DEFAULT_TTL) + ) + } + + private companion object { + const val DEFAULT_TTL = 14L * 24 * 60 * 60 * 1000 + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabaseTest.kt b/app/src/test/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabaseTest.kt new file mode 100644 index 0000000000..ddf26d3b08 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/database/UnsupportedMessageDatabaseTest.kt @@ -0,0 +1,415 @@ +package org.thoughtcrime.securesms.database + +import android.content.Context +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.sqlite.db.SupportSQLiteOpenHelper +import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory +import androidx.test.core.app.ApplicationProvider +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.session.libsession.messaging.messages.UnsupportedMessage.Kind + +/** + * Runs against the real `mms` schema so the placeholder trigger is exercised the way every delete + * path in [MmsDatabase] reaches it. Synthetic payloads only. + */ +@RunWith(RobolectricTestRunner::class) +@Config(minSdk = 36) // Matches SnodeDatabaseTest: the app relies on a recent SQLite +class UnsupportedMessageDatabaseTest { + private lateinit var helper: SupportSQLiteOpenHelper + private lateinit var db: UnsupportedMessageDatabase + + private val sqlite: SupportSQLiteDatabase get() = helper.writableDatabase + + @Before + fun setUp() { + val context = ApplicationProvider.getApplicationContext() + val config = SupportSQLiteOpenHelper.Configuration.builder(context) + .name(null) + .callback(object : SupportSQLiteOpenHelper.Callback(1) { + override fun onCreate(db: SupportSQLiteDatabase) { + db.execSQL(MmsDatabase.CREATE_TABLE) + UnsupportedMessageDatabase.createTable(db) + } + + override fun onUpgrade(db: SupportSQLiteDatabase, oldVersion: Int, newVersion: Int) {} + }) + .build() + + helper = FrameworkSQLiteOpenHelperFactory().create(config) + db = UnsupportedMessageDatabase(helper = { helper }) + } + + private fun insertMmsRow(threadId: Long): Long = + sqlite.compileStatement("INSERT INTO ${MmsDatabase.TABLE_NAME} (${MmsSmsColumns.THREAD_ID}, ${MmsDatabase.DATE_SENT}) VALUES (?, 1)").use { + it.bindLong(1, threadId) + it.executeInsert() + } + + private fun insert( + hash: String, + kind: Kind = Kind.UNKNOWN_TYPE, + size: Int = 10, + placeholderId: Long? = null, + expiresAtMs: Long? = null, + receivedAtMs: Long = 0L, + version: String = "1.0.0-1", + sender: String? = null, + sentTimestampMs: Long? = null, + namespace: Int = 0, + data: ByteArray = ByteArray(size) { it.toByte() }, + ): Boolean = db.insert( + kind = kind, + swarmPublicKey = "05" + "00".repeat(32), + namespace = namespace, + hash = hash, + sender = sender, + sentTimestampMs = sentTimestampMs, + serverTimestampMs = 100L, + serverExpiryMs = null, + data = data, + placeholderMessageId = placeholderId, + expiresAtMs = expiresAtMs, + receivedAtMs = receivedAtMs, + lastAttemptVersion = version, + ) + + private fun idOf(hash: String): Long = + sqlite.query("SELECT id FROM ${UnsupportedMessageDatabase.TABLE_NAME} WHERE hash = ?", arrayOf(hash)) + .use { it.moveToNext(); it.getLong(0) } + + private fun lastAttemptVersion(hash: String): String = + sqlite.query("SELECT last_attempt_version FROM ${UnsupportedMessageDatabase.TABLE_NAME} WHERE hash = ?", arrayOf(hash)) + .use { it.moveToNext(); it.getString(0) } + + /** total_bytes and newer_format_count from the stats row, then the same recomputed from the table */ + private fun assertStatsExact() { + val stats = sqlite.query("SELECT total_bytes, newer_format_count FROM ${UnsupportedMessageDatabase.STATS_TABLE_NAME}").use { + assertTrue(it.moveToNext()) + val result = it.getLong(0) to it.getLong(1) + assertFalse("exactly one stats row", it.moveToNext()) + result + } + val actual = sqlite.query( + "SELECT IFNULL(SUM(length(data) + ?), 0), IFNULL(SUM(kind = 'newerFormat'), 0) FROM ${UnsupportedMessageDatabase.TABLE_NAME}", + arrayOf(UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES) + ).use { it.moveToNext(); it.getLong(0) to it.getLong(1) } + + assertEquals(actual, stats) + } + + private fun stats(): Pair = + sqlite.query("SELECT total_bytes, newer_format_count FROM ${UnsupportedMessageDatabase.STATS_TABLE_NAME}") + .use { it.moveToNext(); it.getLong(0) to it.getLong(1) } + + private fun count(): Long = sqlite.query("SELECT COUNT(*) FROM ${UnsupportedMessageDatabase.TABLE_NAME}") + .use { it.moveToNext(); it.getLong(0) } + + @Test + fun `table has exactly the shared column names`() { + val columns = sqlite.query("PRAGMA table_info(${UnsupportedMessageDatabase.TABLE_NAME})").use { cursor -> + generateSequence { if (cursor.moveToNext()) cursor.getString(cursor.getColumnIndexOrThrow("name")) else null }.toList() + } + + assertEquals( + listOf( + "id", "kind", "swarm_public_key", "namespace", "hash", "sender", "sent_timestamp_ms", + "server_timestamp_ms", "server_expiry_ms", "data", "placeholder_message_id", "expires_at_ms", + "received_at_ms", "last_attempt_version" + ), + columns + ) + } + + @Test + fun `round trips a record and ignores a duplicate hash`() { + assertTrue(insert("hash1", kind = Kind.NEWER_FORMAT, size = 4)) + assertFalse(insert("hash1")) + assertTrue(db.exists("hash1")) + assertFalse(db.exists("hash2")) + + val id = idOf("hash1") + val record = db.get(id)!! + + assertEquals(Kind.NEWER_FORMAT, record.kind) + assertArrayEquals(byteArrayOf(0, 1, 2, 3), record.data) + assertNull(record.placeholderMessageId) + assertNull(record.serverExpiryMs) + assertNull(record.sender) + assertNull(record.sentTimestampMs) + } + + @Test + fun `round trips the sender and sent timestamp`() { + insert("hash1", sender = SENDER_A, sentTimestampMs = 1_234L) + + val record = db.get(idOf("hash1"))!! + + assertEquals(SENDER_A, record.sender) + assertEquals(1_234L, record.sentTimestampMs) + } + + @Test + fun `an unsend deletes only the record with the same sender and sent timestamp`() { + insert("target", sender = SENDER_A, sentTimestampMs = 1_000L) + insert("otherTime", sender = SENDER_A, sentTimestampMs = 2_000L) + insert("otherSender", sender = SENDER_B, sentTimestampMs = 1_000L) + insert("newerFormat", kind = Kind.NEWER_FORMAT) + + db.deleteForUnsend(sender = SENDER_A, sentTimestampMs = 1_000L) + + assertFalse(db.exists("target")) + assertTrue(db.exists("otherTime")) + assertTrue(db.exists("otherSender")) + assertTrue(db.exists("newerFormat")) + } + + @Test + fun `a detached record survives its placeholder being deleted`() { + val placeholder = insertMmsRow(threadId = 1) + insert("hash1", placeholderId = placeholder) + val id = idOf("hash1") + + db.detachPlaceholder(id, expiresAtMs = 5_000L) + sqlite.execSQL("DELETE FROM ${MmsDatabase.TABLE_NAME} WHERE _id = ?", arrayOf(placeholder)) + + val record = db.get(id)!! + assertNull(record.placeholderMessageId) + assertEquals(5_000L, record.expiresAtMs) + } + + @Test + fun `stamping newer format rows marks them attempted without returning them`() { + insert("newer1", kind = Kind.NEWER_FORMAT, version = "1.0.0-1") + insert("newer2", kind = Kind.NEWER_FORMAT, version = "1.0.0-2") + insert("unknown", kind = Kind.UNKNOWN_TYPE, version = "1.0.0-1") + + assertEquals(1, db.stampNewerFormatAttempted("1.0.0-2")) + + assertEquals("1.0.0-2", lastAttemptVersion("newer1")) + assertEquals("1.0.0-2", lastAttemptVersion("newer2")) + assertEquals("1.0.0-1", lastAttemptVersion("unknown")) + } + + @Test + fun `the replay page returns only unknown type rows not attempted by the version`() { + insert("newer", kind = Kind.NEWER_FORMAT, version = "1.0.0-1") + insert("unknown1", version = "1.0.0-1") + insert("attempted", version = "1.0.0-2") + insert("unknown2", version = "1.0.0-1") + insert("unknown3", version = "1.0.0-1") + + assertEquals( + listOf(idOf("unknown1"), idOf("unknown2")), + db.unknownTypeIdsNotAttemptedBy("1.0.0-2", afterId = 0, limit = 2) + ) + assertEquals( + listOf(idOf("unknown3")), + db.unknownTypeIdsNotAttemptedBy("1.0.0-2", afterId = idOf("unknown2"), limit = 2) + ) + + db.setLastAttemptVersion(idOf("unknown1"), "1.0.0-2") + assertEquals( + listOf(idOf("unknown2"), idOf("unknown3")), + db.unknownTypeIdsNotAttemptedBy("1.0.0-2", afterId = 0, limit = 50) + ) + } + + @Test + fun `a group unknown type row starting with a zero byte is replayed, not treated as newer format`() { + insert("group", kind = Kind.UNKNOWN_TYPE, namespace = GROUP_MESSAGES_NAMESPACE, data = byteArrayOf(0, 2, 7, 7)) + + db.stampNewerFormatAttempted("1.0.0-2") + + assertEquals("1.0.0-1", lastAttemptVersion("group")) + assertEquals(listOf(idOf("group")), db.unknownTypeIdsNotAttemptedBy("1.0.0-2", afterId = 0, limit = 50)) + } + + @Test + fun `deleting a placeholder deletes its record`() { + val placeholder = insertMmsRow(threadId = 1) + val other = insertMmsRow(threadId = 1) + insert("hash1", placeholderId = placeholder) + insert("hash2") + + sqlite.execSQL("DELETE FROM ${MmsDatabase.TABLE_NAME} WHERE _id = ?", arrayOf(other)) + assertEquals(2L, count()) + + sqlite.execSQL("DELETE FROM ${MmsDatabase.TABLE_NAME} WHERE _id = ?", arrayOf(placeholder)) + assertFalse(db.exists("hash1")) + assertTrue(db.exists("hash2")) + } + + @Test + fun `deleting a conversation deletes its records and no others`() { + insert("thread1", placeholderId = insertMmsRow(threadId = 1)) + insert("thread1b", placeholderId = insertMmsRow(threadId = 1)) + insert("thread2", placeholderId = insertMmsRow(threadId = 2)) + + // The statement MmsDatabase.deleteThreads issues + sqlite.execSQL("DELETE FROM ${MmsDatabase.TABLE_NAME} WHERE thread_id IN (SELECT value FROM json_each(?))", arrayOf("[1]")) + + assertFalse(db.exists("thread1")) + assertFalse(db.exists("thread1b")) + assertTrue(db.exists("thread2")) + } + + @Test + fun `marking a placeholder deleted for me deletes its record`() { + val placeholder = insertMmsRow(threadId = 1) + insert("hash1", placeholderId = placeholder) + + sqlite.execSQL(UnsupportedMessageDatabase.DELETE_FOR_PLACEHOLDER_SQL, arrayOf(placeholder)) + + assertFalse(db.exists("hash1")) + } + + @Test + fun `expired records are removed`() { + insert("expired", expiresAtMs = 1_000L) + insert("exactly", expiresAtMs = 2_000L) + insert("later", expiresAtMs = 3_000L) + insert("never") + + db.enforceLimits(nowMs = 2_000L) + + assertFalse(db.exists("expired")) + assertFalse(db.exists("exactly")) + assertTrue(db.exists("later")) + assertTrue(db.exists("never")) + } + + @Test + fun `the stats row is seeded and stays exact across inserts, updates and deletes`() { + assertEquals(0L to 0L, stats()) + + val placeholder = insertMmsRow(threadId = 1) + insert("unknown", size = 10, placeholderId = placeholder) + insert("newer1", kind = Kind.NEWER_FORMAT, size = 3) + insert("newer2", kind = Kind.NEWER_FORMAT, size = 0) + assertFalse(insert("newer1", kind = Kind.NEWER_FORMAT, size = 50)) + assertEquals((13L + 3 * UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES) to 2L, stats()) + assertStatsExact() + + sqlite.execSQL("UPDATE ${UnsupportedMessageDatabase.TABLE_NAME} SET kind = 'unknownType', data = ? WHERE hash = 'newer1'", arrayOf(ByteArray(20))) + assertEquals((30L + 3 * UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES) to 1L, stats()) + assertStatsExact() + + db.delete(idOf("newer2")) + assertStatsExact() + + sqlite.execSQL("DELETE FROM ${MmsDatabase.TABLE_NAME} WHERE _id = ?", arrayOf(placeholder)) + assertFalse(db.exists("unknown")) + assertEquals((20L + UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES) to 0L, stats()) + assertStatsExact() + + db.delete(idOf("newer1")) + assertEquals(0L to 0L, stats()) + } + + @Test + fun `the newer format count cap evicts the oldest newer format rows only`() { + insert("unknown", kind = Kind.UNKNOWN_TYPE) + repeat(5) { insert("newer$it", kind = Kind.NEWER_FORMAT) } + + db.enforceRetentionCaps(maxNewerFormatCount = 3) + + assertFalse(db.exists("newer0")) + assertFalse(db.exists("newer1")) + assertTrue(db.exists("newer2")) + assertTrue(db.exists("newer3")) + assertTrue(db.exists("newer4")) + assertTrue(db.exists("unknown")) + assertEquals(3L, stats().second) + assertStatsExact() + } + + @Test + fun `the default newer format count cap is ten thousand`() { + sqlite.execSQL(""" + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 10001) + INSERT INTO ${UnsupportedMessageDatabase.TABLE_NAME} ( + kind, swarm_public_key, namespace, hash, server_timestamp_ms, data, received_at_ms, last_attempt_version + ) + SELECT 'newerFormat', '05', 0, 'h' || i, 0, x'00', 0, '1.0.0-1' FROM n + """) + + db.enforceRetentionCaps() + + assertEquals(10_000L, count()) + assertFalse(db.exists("h1")) + assertTrue(db.exists("h2")) + assertStatsExact() + } + + @Test + fun `the byte budget counts the per row overhead, so many tiny rows are evicted`() { + repeat(250) { insert("tiny$it", size = 1) } + val rowCost = 1 + UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES + + // Their data alone (250 bytes) fits easily + db.enforceRetentionCaps(maxRetainedBytes = 120 * rowCost) + + assertEquals(120L, count()) + assertFalse(db.exists("tiny129")) + assertTrue(db.exists("tiny130")) + assertStatsExact() + } + + @Test + fun `the byte budget evicts newer format rows first, oldest first, until within budget`() { + insert("unknownOld", kind = Kind.UNKNOWN_TYPE, size = 10) + insert("newerOld", kind = Kind.NEWER_FORMAT, size = 10) + insert("unknownNew", kind = Kind.UNKNOWN_TYPE, size = 10) + insert("newerNew", kind = Kind.NEWER_FORMAT, size = 10) + val rowCost = 10 + UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES + + db.enforceRetentionCaps(maxRetainedBytes = 4 * rowCost) + assertEquals(4L, count()) + + db.enforceRetentionCaps(maxRetainedBytes = 3 * rowCost) + assertFalse(db.exists("newerOld")) + assertTrue(db.exists("newerNew")) + + db.enforceRetentionCaps(maxRetainedBytes = 2 * rowCost + 1) + assertFalse(db.exists("newerNew")) + assertTrue(db.exists("unknownOld")) + assertTrue(db.exists("unknownNew")) + + db.enforceRetentionCaps(maxRetainedBytes = rowCost) + assertFalse(db.exists("unknownOld")) + assertTrue(db.exists("unknownNew")) + assertStatsExact() + } + + @Test + fun `maintenance removes expired rows before applying the byte budget`() { + insert("unknownOld", kind = Kind.UNKNOWN_TYPE, size = 10) + insert("newerExpired", kind = Kind.NEWER_FORMAT, size = 10, expiresAtMs = 1_000L) + insert("unknownNew", kind = Kind.UNKNOWN_TYPE, size = 10) + + db.enforceLimits(nowMs = 1_000L, maxRetainedBytes = 2L * (10 + UnsupportedMessageDatabase.ROW_OVERHEAD_BYTES)) + + assertFalse(db.exists("newerExpired")) + assertTrue(db.exists("unknownOld")) + assertTrue(db.exists("unknownNew")) + assertStatsExact() + } + + private companion object { + val SENDER_A = "05" + "aa".repeat(32) + val SENDER_B = "05" + "bb".repeat(32) + + // Namespace.GROUP_MESSAGES(), which can't be read here as libsession's native library + // doesn't load in JVM tests + const val GROUP_MESSAGES_NAMESPACE = 11 + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContentTest.kt b/app/src/test/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContentTest.kt new file mode 100644 index 0000000000..598a6a8571 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/database/model/content/UnsupportedMessageContentTest.kt @@ -0,0 +1,19 @@ +package org.thoughtcrime.securesms.database.model.content + +import kotlinx.serialization.json.Json +import org.junit.Assert.assertEquals +import org.junit.Test + +class UnsupportedMessageContentTest { + private val json = Json { + serializersModule = MessageContentModule().provideMessageContentSerializersModule() + } + + @Test + fun `round trips through the message_content column format`() { + val encoded = json.encodeToString(UnsupportedMessageContent) + + assertEquals("""{"type":"unsupported_message"}""", encoded) + assertEquals(UnsupportedMessageContent, json.decodeFromString(encoded)) + } +}