Support role=application web widgets

After: The green rect follows web focus. Before: TalkBack tries to read the TV apps as text documents. Background: Within <xxx role=application>...</xxx>, screen readers should not consume DPAD (arrow key) events. Web apps or widgets with role=application have, per the WAI-ARIA spec's contract, their own JavaScript logic for moving focus [1]. [1] w3c/aria#1049, where we discussed this. Problem: TalkBack does not handle role=application so such web apps lose their 4-way (up/down/left/right) navigation. TalkBack only moves forward/backward which breaks authors' pre-defined TV UX. Solution: Whenever accessibility focus (the green rect) goes to some web content with <body role=application> or anywhere within a role=application widget, we don't consume the DPAD events; we let them through. Testing done: Open a simple TV web app that has <body role=application>. Notice: Once the web view gets accessibilty focus, TalkBack won't eat (consume) DPAD key events and the the key events reach the web page's key handler in JavaScript.
google · Dec 3, 2021 · bc2c99a · bc2c99a
1 parent 9db811a
commit bc2c99a
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 0 deletions.
diff --git a/.../com/google/android/accessibility/talkback/controller/TelevisionNavigationController.java b/.../com/google/android/accessibility/talkback/controller/TelevisionNavigationController.java
@@ -361,6 +361,13 @@ private boolean shouldHandleEvent(AccessibilityNodeInfoCompat cursor, KeyEvent e
         return false;
       }
     }
+
+    // Web applications and web widgets with role=application have, per the
+    // WAI-ARIA spec's contract, their own JavaScript logic for moving focus.
+    // TalkBack should not consume key events when such an app has accessibility focus.
+    boolean shouldProcessDPadKeyEvent = this.shouldProcessDPadKeyEvent &&
+            !AccessibilityNodeInfoUtils.isWebApplication(cursor);
+
     // TalkBack should always consume up/down/left/right on the d-pad, unless
     // shouldProcessDPadKeyEvent is false. Otherwise, strange things will happen when TalkBack
     // cannot navigate further.

diff --git a/utils/src/main/java/com/google/android/accessibility/utils/AccessibilityNodeInfoUtils.java b/utils/src/main/java/com/google/android/accessibility/utils/AccessibilityNodeInfoUtils.java
@@ -381,6 +381,39 @@ public boolean accept(AccessibilityNodeInfoCompat node) {
                 || (node != null && node.getCollectionInfo() != null);
           });
 
+  public static boolean hasApplicationWebRole(AccessibilityNodeInfoCompat node) {
+    return node != null && node.getExtras() != null
+            && node.getExtras().containsKey("AccessibilityNodeInfo.chromeRole")
+            && node.getExtras().get("AccessibilityNodeInfo.chromeRole").equals("application");
+  }
+
+  private static final Filter<AccessibilityNodeInfoCompat> FILTER_IN_WEB_APPLICATION =
+          new Filter<AccessibilityNodeInfoCompat>() {
+            @Override
+            public boolean accept(AccessibilityNodeInfoCompat node) {
+              return hasApplicationWebRole(node);
+            }
+          };
+
+  /**
+   * Returns true if |node| has role=application, i.e. |node| has JavaScript
+   * that handles key events.
+   */
+  public static boolean isWebApplication(AccessibilityNodeInfoCompat node) {
+      // When a WebView-like view (an actual WebView or a browser) has focus:
+      // Check the web content's accessibility tree's first node.
+      // If that node wants raw key event, instead of first "tabbing" the green
+      // rect to it, skip ahead and let the web app directly decide where to go.
+      boolean firstWebNode = WebInterfaceUtils.supportsWebActions(node)
+              && !WebInterfaceUtils.supportsWebActions(node.getParent());
+      boolean firstWebNodeWantsKeyEvents = firstWebNode
+              && node.getChildCount() > 0
+              && hasApplicationWebRole(node.getChild(0));
+
+      return firstWebNodeWantsKeyEvents
+              || getSelfOrMatchingAncestor(node, FILTER_IN_WEB_APPLICATION) != null;
+    }
+
   private AccessibilityNodeInfoUtils() {
     // This class is not instantiable.
   }