From b029da0d0ba6d994ff63753e20f28092572e9c75 Mon Sep 17 00:00:00 2001 From: willbilec Date: Thu, 8 Oct 2026 14:13:59 -0600 Subject: [PATCH] Read Samsung watch notification content before the timestamp --- differences.md | 4 + .../SamsungNotificationDescription.kt | 153 ++++++++++++++ .../roledescription/TreeNodesDescription.java | 11 ++ ...eViewAccessibilityFocusedFeedbackRule.java | 3 + .../SamsungNotificationDescriptionTest.kt | 187 ++++++++++++++++++ 5 files changed, 358 insertions(+) create mode 100644 talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescription.kt create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescriptionTest.kt diff --git a/differences.md b/differences.md index a6c411d72..0159e1dfe 100644 --- a/differences.md +++ b/differences.md @@ -152,6 +152,10 @@ When you turn Backtalk off, it says "Backtalk off" at the accessibility volume, ## Notifications +### Samsung watch notification content + +On Samsung watches, Backtalk reads the app and title, then the message content, then the time when you focus a notification card. Samsung's card label can omit the message even though it is available in the accessibility tree. Backtalk includes that text automatically, without an extra gesture or setting, and does not announce the card's display font formatting. + ### Speak notifications setting To stop Backtalk from reading new notifications when they arrive, turn off **Verbosity** > **Speak notifications**. Backtalk still reads incoming calls, and you can still read notifications in the notification shade. The setting is on by default. diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescription.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescription.kt new file mode 100644 index 000000000..4fc0d6fd2 --- /dev/null +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescription.kt @@ -0,0 +1,153 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.accessibility.talkback.compositor.roledescription + +import android.content.Context +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat +import com.google.android.accessibility.talkback.compositor.AccessibilityNodeFeedbackUtils +import com.google.android.accessibility.talkback.compositor.GlobalVariables +import com.google.android.accessibility.utils.AccessibilityNodeInfoUtils +import com.google.android.accessibility.utils.FormFactorUtils + +/** Supplements Samsung's labelled notification card with the message its label omits. */ +object SamsungNotificationDescription { + internal const val PACKAGE = "com.samsung.android.wearable.sysui" + internal const val CARD_ID = "$PACKAGE:id/chunk_view" + internal const val BODY_ID = "$PACKAGE:id/single_body_text" + internal const val TIME_ID = "$PACKAGE:id/timestamp" + + @JvmStatic + fun isNotificationCard(node: AccessibilityNodeInfoCompat): Boolean = + isNotificationCard(node.packageName?.toString(), node.viewIdResourceName, + FormFactorUtils.isAndroidWear()) + + internal fun isNotificationCard(packageName: String?, resourceId: String?, isWear: Boolean) = + isWear && packageName == PACKAGE && resourceId == CARD_ID + + /** A lazy tree view so extraction can be tested without Android accessibility objects. */ + internal interface Node { + val packageName: String? + val resourceId: String? + val text: CharSequence? + val visible: Boolean + val focusable: Boolean + val children: Sequence + fun speech(): CharSequence + } + + @JvmStatic + fun describeCard( + node: AccessibilityNodeInfoCompat, + announcement: CharSequence, + shouldIterateChildren: Boolean, + context: Context, + globalVariables: GlobalVariables, + ): List? { + // Avoid accessing child nodes at all on phones and unrelated watch screens. + if (!shouldIterateChildren || !isNotificationCard(node)) return null + + class AndroidNode(private val value: AccessibilityNodeInfoCompat) : Node { + override val packageName get() = value.packageName?.toString() + override val resourceId get() = value.viewIdResourceName + override val text get() = value.text + override val visible get() = AccessibilityNodeInfoUtils.isVisible(value) + override val focusable get() = AccessibilityNodeInfoUtils.isAccessibilityFocusable(value) + override val children get() = (0 until value.childCount).asSequence() + .mapNotNull { value.getChild(it) }.map { AndroidNode(it) } + override fun speech() = + AccessibilityNodeFeedbackUtils.getNodeTextDescription(value, context, globalVariables) + } + + return describeCard(AndroidNode(node), announcement, true, true) + } + + internal fun describeCard( + card: Node, + announcement: CharSequence, + isWear: Boolean, + shouldIterateChildren: Boolean, + ): List? { + if (!isNotificationCard(card.packageName, card.resourceId, isWear) || + !shouldIterateChildren || normalize(announcement).isEmpty() + ) return null + + val bodies = collectBodies(card, announcement, isWear, shouldIterateChildren) + val timestamp = readableDescendants(card).firstOrNull { + it.resourceId == TIME_ID && normalize(it.text ?: "").isNotEmpty() + } + if (timestamp == null) return listOf(announcement) + bodies + + // Match the actual timestamp node, including localized time and Unicode spacing. + // Only remove a trailing time: a matching time mentioned in the title/message stays intact. + val rawTime = normalize(timestamp.text ?: "") + val timePattern = rawTime.split(' ').joinToString("[\\s\\p{Z}]+") { Regex.escape(it) } + val suffix = Regex("(?:[,;][\\s\\p{Z}]*|[\\s\\p{Z}]+)$timePattern[\\s\\p{Z}]*$") + .find(announcement) + if (suffix == null && normalize(announcement).contains(rawTime)) { + return listOf(announcement) + bodies + } + val header = if (suffix == null) announcement else + announcement.subSequence(0, suffix.range.first).trimEnd() + val timeSpeech = timestamp.speech() + if (normalize(timeSpeech).isEmpty()) return listOf(announcement) + bodies + return listOf(header) + bodies + timeSpeech + } + + internal fun collectBodies( + card: Node, + announcement: CharSequence, + isWear: Boolean, + shouldIterateChildren: Boolean, + ): List { + if (!isNotificationCard(card.packageName, card.resourceId, isWear) || + !shouldIterateChildren || normalize(announcement).isEmpty() + ) return emptyList() + + val bodies = mutableListOf() + val spoken = mutableListOf(normalize(announcement)) + readableDescendants(card).forEach { node -> + if (node.resourceId == BODY_ID) { + val body = normalize(node.text ?: "") + if (body.isNotEmpty() && spoken.none { it.contains(body) }) { + val speech = node.speech() + if (normalize(speech).isNotEmpty()) { + bodies.add(speech) + spoken.add(body) + } + } + } + } + return bodies + } + + private fun readableDescendants(card: Node): Sequence { + fun visit(node: Node, depth: Int): Sequence = sequence { + // Never cross into another card, hidden subtree, or independently navigable control. + if (depth > 32 || node.packageName != PACKAGE || node.resourceId == CARD_ID || + !node.visible || node.focusable + ) return@sequence + yield(node) + if (node.resourceId != BODY_ID && node.resourceId != TIME_ID) { + node.children.forEach { yieldAll(visit(it, depth + 1)) } + } + } + return card.children.flatMap { visit(it, 0) } + } + + private fun normalize(text: CharSequence): String = + text.toString().replace(Regex("[\\s\\p{Z}]+"), " ").trim() +} diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/TreeNodesDescription.java b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/TreeNodesDescription.java index e3e5b02e8..b8695176f 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/TreeNodesDescription.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/roledescription/TreeNodesDescription.java @@ -296,6 +296,17 @@ private CharSequence treeNodesDescription( TextUtils.isEmpty( AccessibilityNodeFeedbackUtils.getNodeContentDescription( node, context, globalVariables)); + // Samsung labels the notification card with its title but leaves the message in a child. + // Read the body before the time; general child aggregation would repeat title/time. + if (!isContentDescriptionEmpty) { + List notificationDescription = + SamsungNotificationDescription.describeCard( + node, joinList.get(0), shouldIterateChildren, context, globalVariables); + if (notificationDescription != null) { + joinList.clear(); + joinList.addAll(notificationDescription); + } + } StringBuilder logString = new StringBuilder(); logString .append(String.format(" (%s)", node.hashCode())) diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/rule/EventTypeViewAccessibilityFocusedFeedbackRule.java b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/rule/EventTypeViewAccessibilityFocusedFeedbackRule.java index 2833dae23..46b0b7ac6 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/rule/EventTypeViewAccessibilityFocusedFeedbackRule.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/compositor/rule/EventTypeViewAccessibilityFocusedFeedbackRule.java @@ -43,6 +43,7 @@ import com.google.android.accessibility.talkback.compositor.EventFeedback; import com.google.android.accessibility.talkback.compositor.EventInterpretation; import com.google.android.accessibility.talkback.compositor.GlobalVariables; +import com.google.android.accessibility.talkback.compositor.roledescription.SamsungNotificationDescription; import com.google.android.accessibility.talkback.compositor.roledescription.TreeNodesDescription; import com.google.android.accessibility.talkback.controlsounds.ControlSounds; import com.google.android.accessibility.talkback.eventprocessor.ProcessorPhoneticLetters; @@ -480,6 +481,8 @@ private static boolean supportInlineFormatting( || (accessibilityFocusEventInterpretation.getEvent() == Compositor.EVENT_UNKNOWN && WebInterfaceUtils.isWebContainer(node)); return assumeIsNavigateByUser + // Samsung's notification rendering styles are not message formatting to announce. + && !SamsungNotificationDescription.isNotificationCard(node) && (!WebInterfaceUtils.isWebContainer(node) || SpannableUtils.hasTargetSpan(node.getText(), AbsoluteSizeSpan.class, false)); } diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescriptionTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescriptionTest.kt new file mode 100644 index 000000000..c07efc090 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/compositor/roledescription/SamsungNotificationDescriptionTest.kt @@ -0,0 +1,187 @@ +/* + * Copyright 2026 Backtalk contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.accessibility.talkback.compositor.roledescription + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class SamsungNotificationDescriptionTest { + private data class FakeNode( + override val packageName: String? = SamsungNotificationDescription.PACKAGE, + override val resourceId: String? = null, + override val text: CharSequence? = null, + override val visible: Boolean = true, + override val focusable: Boolean = false, + val nodes: List = emptyList(), + val processedText: CharSequence? = text, + ) : SamsungNotificationDescription.Node { + override val children get() = nodes.asSequence() + override fun speech() = processedText ?: "" + } + + private fun body(text: String) = + FakeNode(resourceId = SamsungNotificationDescription.BODY_ID, text = text) + + private fun card(vararg nodes: FakeNode) = + FakeNode(resourceId = SamsungNotificationDescription.CARD_ID, focusable = true, + nodes = nodes.toList()) + + private fun collect( + node: FakeNode, + announcement: String = "WhatsApp, Group, 9:11 p.m.", + wear: Boolean = true, + iterate: Boolean = true, + ) = SamsungNotificationDescription.collectBodies(node, announcement, wear, iterate) + .map { it.toString() } + + @Test fun addsOnlyBodyAndPreservesProcessing() { + val message = body("Hello 😁").copy(processedText = "Hello beaming face") + val title = FakeNode(resourceId = "${SamsungNotificationDescription.PACKAGE}:id/title", + text = "Group") + val time = FakeNode(resourceId = "${SamsungNotificationDescription.PACKAGE}:id/timestamp", + text = "9:11 p.m.") + val action = FakeNode(text = "Reply", focusable = true) + assertEquals(listOf("Hello beaming face"), collect(card(title, time, message, action))) + } + + @Test fun skipsBodyAlreadyInAnnouncementWithUnicodeWhitespace() { + assertTrue(collect(card(body("Hello\u00a0 world")), "Group, Hello world").isEmpty()) + } + + @Test fun skipsRepeatedBodiesWithoutChangingTheirWording() { + assertEquals(listOf(" Hello\u00a0world "), + collect(card(body(" Hello\u00a0world "), body("Hello world")))) + } + + @Test fun skipsEmptyMissingAndUnavailableBody() { + assertTrue(collect(card(body(" \n\u00a0"), + FakeNode(resourceId = SamsungNotificationDescription.BODY_ID))).isEmpty()) + assertTrue(collect(card()).isEmpty()) + } + + @Test fun readsNestedBodiesInTreeOrder() { + assertEquals(listOf("First", "Second"), + collect(card(FakeNode(nodes = listOf(body("First"))), body("Second")))) + } + + @Test fun skipsHiddenBodyAndHiddenAncestor() { + assertTrue(collect(card(body("Hidden").copy(visible = false), + FakeNode(visible = false, nodes = listOf(body("Also hidden"))))).isEmpty()) + } + + @Test fun skipsFocusableBodyAndFocusableAncestor() { + assertTrue(collect(card(body("Separate").copy(focusable = true), + FakeNode(focusable = true, nodes = listOf(body("Separate child"))))).isEmpty()) + } + + @Test fun neverReadsNeighboringOrNestedCards() { + val first = card(body("Current"), card(body("Nested other card"))) + val second = card(body("Neighbor")) + assertEquals(listOf("Current"), collect(first)) + assertTrue(collect(FakeNode(nodes = listOf(first, second))).isEmpty()) + } + + @Test fun rejectsOtherPackagesAndCardIds() { + assertTrue(collect(card(body("Message")).copy(packageName = "another.app")).isEmpty()) + assertTrue(collect(card(body("Message")).copy(resourceId = "android:id/content")).isEmpty()) + assertTrue(collect(card(body("Message").copy(packageName = "another.app"))).isEmpty()) + } + + @Test fun phoneAndDisabledChildIterationDoNotReadChildren() { + val unreadable = object : SamsungNotificationDescription.Node { + override val packageName = SamsungNotificationDescription.PACKAGE + override val resourceId = SamsungNotificationDescription.CARD_ID + override val text = "" + override val visible = true + override val focusable = true + override val children: Sequence + get() = error("Must not access children") + override fun speech() = "" + } + assertTrue(SamsungNotificationDescription.collectBodies(unreadable, "Group", false, true) + .isEmpty()) + assertTrue(SamsungNotificationDescription.collectBodies(unreadable, "Group", true, false) + .isEmpty()) + } + + @Test fun unlabelledCardsRetainNormalAggregation() { + assertTrue(collect(card(body("Message")), "").isEmpty()) + } + + private fun time(text: String) = + FakeNode(resourceId = SamsungNotificationDescription.TIME_ID, text = text) + + private fun describe(node: FakeNode, announcement: String = "WhatsApp, Group, 9:11 p.m.") = + SamsungNotificationDescription.describeCard(node, announcement, true, true) + ?.map { it.toString() } + + @Test fun readsHeaderThenBodyThenTimeOnce() { + val timestamp = time("9:11\u202fp.m.").copy(processedText = "9:11 p.m.") + assertEquals(listOf("WhatsApp, Group", "Message", "9:11 p.m."), + describe(card(timestamp, body("Message")), "WhatsApp, Group , 9:11\u202fp.m.")) + } + + @Test fun handlesLocalizedTimeAndKeepsTimesInsideTitleAndBody() { + assertEquals(listOf("WhatsApp, Meet at 21:11", "I arrive at 21:11", "21:11"), + describe(card(time("21:11"), body("I arrive at 21:11")), + "WhatsApp, Meet at 21:11, 21:11")) + } + + @Test fun appendsTimeAfterBodyWhenLabelDoesNotContainIt() { + assertEquals(listOf("WhatsApp, Group", "Message", "Yesterday"), + describe(card(time("Yesterday"), body("Message")), "WhatsApp, Group")) + } + + @Test fun missingOrHiddenTimeKeepsLabelAndBody() { + assertEquals(listOf("WhatsApp, Group, 9:11 p.m.", "Message"), + describe(card(body("Message")))) + assertEquals(listOf("WhatsApp, Group", "Message"), + describe(card(time("9:11 p.m.").copy(visible = false), body("Message")), + "WhatsApp, Group")) + } + + @Test fun doesNotDuplicateBodyAlreadyIncludedBeforeTime() { + assertEquals(listOf("WhatsApp, Group, Message", "9:11 p.m."), + describe(card(time("9:11 p.m."), body("Message")), + "WhatsApp, Group, Message, 9:11 p.m.")) + } + + @Test fun missingBodyStillReadsTimeOnce() { + assertEquals(listOf("WhatsApp, Group", "9:11 p.m."), + describe(card(time("9:11 p.m.")))) + } + + @Test fun reorderedDescriptionIsOnlyForSupportedWearCards() { + val node = card(time("9:11 p.m."), body("Message")) + assertEquals(null, SamsungNotificationDescription.describeCard(node, "Group", false, true)) + assertEquals(null, SamsungNotificationDescription.describeCard(node, "Group", true, false)) + assertEquals(null, describe(node.copy(packageName = "other.app"))) + assertEquals(null, describe(node.copy(resourceId = "android:id/content"))) + } + + @Test fun fontAnnouncementExemptionIsOnlyForSamsungWearNotificationCard() { + assertTrue(SamsungNotificationDescription.isNotificationCard( + SamsungNotificationDescription.PACKAGE, SamsungNotificationDescription.CARD_ID, true)) + assertEquals(false, SamsungNotificationDescription.isNotificationCard( + SamsungNotificationDescription.PACKAGE, SamsungNotificationDescription.CARD_ID, false)) + assertEquals(false, SamsungNotificationDescription.isNotificationCard( + SamsungNotificationDescription.PACKAGE, SamsungNotificationDescription.BODY_ID, true)) + assertEquals(false, SamsungNotificationDescription.isNotificationCard( + "other.app", SamsungNotificationDescription.CARD_ID, true)) + } +}