From 18a3d455eb2a095c6237b906ab49953ebdb8d211 Mon Sep 17 00:00:00 2001 From: Hugo Holgersson Date: Thu, 19 Sep 2019 14:37:26 +0200 Subject: [PATCH] Support TV web apps in WebViews Background: Within ..., screen readers should not consume DPAD (arrow key) events. Web apps or widgets with role=application have, per the WAI-ARIA spec's contract, their own JavaScript logic for moving focus [1]. [1] https://github.com/w3c/aria/issues/1049, where we discussed this. Problem: TalkBack does not handle role=application so such web apps lose their 4-way (up/down/left/right) navigation. TalkBack only moves foward/backward which breaks authors' pre-defined TV UX. Solution: Whenever accessibility focus (the green rect) goes to a WebView with or anywhere within a role=application widget, we don't consume the DPAD events; we let them through. Testing done: From TalkBack's test app, open dpad_a11y.html which has . Notice: I. Once the WebView gets accessibilty focus, TalkBack won't consume DPAD key events. II. The key events reach the web page's key handlers in JavaScript. III: TalkBack describes HTML elements once they get focused. --- .../TelevisionNavigationController.java | 9 ++ testapp/app/src/main/assets/dpad_a11y.html | 106 ++++++++++++++++++ .../testsession/WebViewDPADTest.java | 54 +++++++++ testapp/app/src/main/res/raw/test.json | 11 ++ .../donottranslate_title_and_description.xml | 6 + .../main/java/AccessibilityNodeInfoUtils.java | 31 +++++ 6 files changed, 217 insertions(+) create mode 100644 testapp/app/src/main/assets/dpad_a11y.html create mode 100644 testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java diff --git a/talkback/src/main/java/controller/TelevisionNavigationController.java b/talkback/src/main/java/controller/TelevisionNavigationController.java index 4d1c39fdb..31c90b64e 100644 --- a/talkback/src/main/java/controller/TelevisionNavigationController.java +++ b/talkback/src/main/java/controller/TelevisionNavigationController.java @@ -343,6 +343,15 @@ public boolean processWhenServiceSuspended() { */ private boolean shouldIgnore(AccessibilityNodeInfoCompat node, KeyEvent event) { final int keyCode = event.getKeyCode(); + if (AccessibilityNodeInfoUtils.isWebApplication(node)) { + // Web applications and web widgets with role=application have, per the + // WAI-ARIA spec's contract, their own JavaScript logic for moving focus. + // TalkBack should not consume key events when such an app has accessibility focus. + // Debug tip: Forward DPAD events whenever the accessibility cursor is on, + // or inside, a WebView: if (WebInterfaceUtils.supportsWebActions(node)) return true; + return true; + } + if (!mShouldProcessDPadKeyEvent && (keyCode == KeyEvent.KEYCODE_DPAD_UP || keyCode == KeyEvent.KEYCODE_DPAD_DOWN diff --git a/testapp/app/src/main/assets/dpad_a11y.html b/testapp/app/src/main/assets/dpad_a11y.html new file mode 100644 index 000000000..5836823d8 --- /dev/null +++ b/testapp/app/src/main/assets/dpad_a11y.html @@ -0,0 +1,106 @@ + + + + Example TV web app + + + + +

This web app allows navigation in 4 directions. Use your arrow keys (DPAD_UP / DPAD_DOWN / DPAD_LEFT / DPAD_RIGHT).

+

role=application makes TalkBack forward DPAD events to Chromium.

+
+
+
A
C
+
+
+
B
D
+
+
+ + + + + + diff --git a/testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java b/testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java new file mode 100644 index 000000000..c876763f4 --- /dev/null +++ b/testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.talkbacktests.testsession; + +import android.content.Context; +import android.view.KeyEvent; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.webkit.WebView; + +import com.android.talkbacktests.R; + +public class WebViewDPADTest extends BaseTestContent { + + public WebViewDPADTest(Context context, String subtitle, String description) { + super(context, subtitle, description); + } + + @Override + public View getView(final LayoutInflater inflater, ViewGroup container, Context context) { + final View view = inflater.inflate(R.layout.test_web_view, container, false); + final WebView webView = (WebView) view.findViewById(R.id.webview); + webView.getSettings().setJavaScriptEnabled(true); + webView.loadUrl("file:///android_asset/dpad_a11y.html"); + webView.setOnKeyListener(new View.OnKeyListener() { + @Override + public boolean onKey(View v, int keyCode, KeyEvent event) { + if (keyCode == KeyEvent.KEYCODE_BACK) { // Exit the WebView + View parent = (View) v.getParent().getParent().getParent(); + View nextButton = parent.findViewById(R.id.next); + // Move Android focus to the native button. + return nextButton != null && nextButton.requestFocus(); + } + return false; + } + }); + return view; + } +} \ No newline at end of file diff --git a/testapp/app/src/main/res/raw/test.json b/testapp/app/src/main/res/raw/test.json index 07807a51f..18ccb98af 100644 --- a/testapp/app/src/main/res/raw/test.json +++ b/testapp/app/src/main/res/raw/test.json @@ -153,6 +153,17 @@ } ] }, + { + "title": "@string/dpad_web_view_session_title", + "description": "@string/dpad_web_view_session_description", + "content": [ + { + "subtitle": "@string/dpad_web_view_test_subtitle", + "description": "@string/dpad_web_view_test_description", + "classname": "com.android.talkbacktests.testsession.WebViewDPADTest" + } + ] + }, { "title": "@string/single_line_edit_Field_session_title", "description": "@string/single_line_edit_Field_session_description", diff --git a/testapp/app/src/main/res/values/donottranslate_title_and_description.xml b/testapp/app/src/main/res/values/donottranslate_title_and_description.xml index 931fb7fc4..6c5fb1005 100644 --- a/testapp/app/src/main/res/values/donottranslate_title_and_description.xml +++ b/testapp/app/src/main/res/values/donottranslate_title_and_description.xml @@ -63,6 +63,9 @@ WebView View with web content + WebView TV app + A TV app where the DPAD moves web focus in four directions. + Single Line Edit Field Single-line editable text field @@ -171,6 +174,9 @@ View with Web Content With TalkBack: swipe between web elements, double-tap to activate, swipe up and down to select element type. Note: requires Chrome 50 or above to work. + WebView TV app + With TalkBack on Android TV: use the directional keys, the DPAD, to move between the four grid items. This test covers TV web apps that implement custom DPAD navigation, for example, TV apps that display grids of video clips. + WebView Inside ScrollView With TalkBack: swipe between web elements until the view scrolls, double-tap to activate. Note: requires Chrome 50 or above to work. diff --git a/utils/src/main/java/AccessibilityNodeInfoUtils.java b/utils/src/main/java/AccessibilityNodeInfoUtils.java index 33dcb50c6..ba6d6a96d 100644 --- a/utils/src/main/java/AccessibilityNodeInfoUtils.java +++ b/utils/src/main/java/AccessibilityNodeInfoUtils.java @@ -148,6 +148,37 @@ public boolean accept(AccessibilityNodeInfoCompat node) { } }; + public static boolean hasApplicationWebRole(AccessibilityNodeInfoCompat node) { + return node != null && node.getExtras() != null + && node.getExtras().containsKey("AccessibilityNodeInfo.chromeRole") + && node.getExtras().get("AccessibilityNodeInfo.chromeRole").equals("application"); + } + + private static final Filter FILTER_IN_WEB_APPLICATION = + new Filter() { + @Override + public boolean accept(AccessibilityNodeInfoCompat node) { + return hasApplicationWebRole(node); + } + }; + + /** + * Returns true if |node| has role=application, i.e. |node| has JavaScript + * that handles key events. + */ + public static boolean isWebApplication(AccessibilityNodeInfoCompat node) { + // When a WebView has focus: Check Chromium's accessibility tree's first node. + // If that node wants raw key event, instead of first "tabbing" the green + // rect to it, skip ahead and let the web app directly decide where to go. + boolean firstChromiumNodeWantsKeyEvents = + Role.getRole(node) == Role.ROLE_WEB_VIEW + && node.getChildCount() > 0 + && hasApplicationWebRole(node.getChild(0)); + + return firstChromiumNodeWantsKeyEvents + || AccessibilityNodeInfoUtils.getSelfOrMatchingAncestor(node, FILTER_IN_WEB_APPLICATION) != null; + } + private AccessibilityNodeInfoUtils() { // This class is not instantiable. }