Commit e6dd62f7 authored by dmazzoni's avatar dmazzoni Committed by Commit bot

Select-to-speak extension code

This is an initial implementation of Select-to-speak. Given mouse events
that are "captured" - currently by holding down Search - it draws a box
around the area the user selects, and then speaks everything within that
box based on its accessible name.

This is not meant to be a final, complete implementation, just a good
starting point to begin experimenting.

Depends on: https://codereview.chromium.org/2493923002/

BUG=593887
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:closure_compilation

Review-Url: https://codereview.chromium.org/2509883002
Cr-Commit-Position: refs/heads/master@{#435555}
parent f05a6730
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
{
'targets': [
{
'target_name': 'chrome_extension_externs',
'includes': ['../../../../../../third_party/closure_compiler/compile_js2.gypi'],
},
],
}
......@@ -41,14 +41,18 @@ def ChromeRootPath(path='.'):
return os.path.relpath(os.path.join(_CHROME_SOURCE_DIR, path))
# Name of chrome extensions externs file.
_CHROME_EXTENSIONS_EXTERNS = (
ChromeRootPath('third_party/closure_compiler/externs/chrome_extensions.js'))
# Automation API externs file.
_AUTOMATION_EXTERNS = (
ChromeRootPath('third_party/closure_compiler/externs/automation.js'))
# MetricsPrivate externs file.
_METRICS_PRIVATE_EXTERNS = (
ChromeRootPath('third_party/closure_compiler/externs/metrics_private.js'))
# Additional chrome extension api externs file.
_CHROME_EXTENSIONS_EXTERNS = (
ChromeRootPath('third_party/closure_compiler/externs/chrome_extensions.js'))
# Externs common to many ChromeVox scripts.
_COMMON_EXTERNS = [
......@@ -57,6 +61,7 @@ _COMMON_EXTERNS = [
CVoxPath('chromevox/background/externs.js'),
CVoxPath('chromevox/injected/externs.js'),
CVoxPath('host/chrome/externs.js'),
_AUTOMATION_EXTERNS,
_CHROME_EXTENSIONS_EXTERNS,
_METRICS_PRIVATE_EXTERNS]
......
......@@ -25,7 +25,7 @@ run_jsbundler("select_to_speak_copied_files") {
mode = "copy"
dest_dir = select_to_speak_out_dir
sources = [
"background.js",
"select_to_speak.js",
]
rewrite_rules = [
rebase_path(".", root_build_dir) + ":",
......
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
{
'targets': [
{
'target_name': 'select_to_speak',
'dependencies': [
'<(EXTERNS_GYP):accessibility_private',
'<(EXTERNS_GYP):automation',
'<(EXTERNS_GYP):chrome_extensions',
],
'includes': ['../../../../../third_party/closure_compiler/compile_js2.gypi'],
},
],
}
......@@ -11,11 +11,12 @@
{% endif %}
"background": {
"scripts": [
"background.js"
"select_to_speak.js"
]
},
"permissions": [
"accessibilityPrivate"
"accessibilityPrivate",
"tts"
],
"automation": {
"desktop": true
......
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
var AutomationEvent = chrome.automation.AutomationEvent;
var AutomationNode = chrome.automation.AutomationNode;
var EventType = chrome.automation.EventType;
var RoleType = chrome.automation.RoleType;
/**
* Return the rect that encloses two points.
* @param {number} x1 The first x coordinate.
* @param {number} y1 The first y coordinate.
* @param {number} x2 The second x coordinate.
* @param {number} y2 The second x coordinate.
* @return {{left: number, top: number, width: number, height: number}}
*/
function rectFromPoints(x1, y1, x2, y2) {
var left = Math.min(x1, x2);
var right = Math.max(x1, x2);
var top = Math.min(y1, y2);
var bottom = Math.max(y1, y2);
return {left: left,
top: top,
width: right - left,
height: bottom - top};
}
/**
* Returns true if |rect1| and |rect2| overlap. The rects must define
* left, top, width, and height.
* @param {{left: number, top: number, width: number, height: number}} rect1
* @param {{left: number, top: number, width: number, height: number}} rect2
* @return {boolean} True if the rects overlap.
*/
function overlaps(rect1, rect2) {
var l1 = rect1.left;
var r1 = rect1.left + rect1.width;
var t1 = rect1.top;
var b1 = rect1.top + rect1.height;
var l2 = rect2.left;
var r2 = rect2.left + rect2.width;
var t2 = rect2.top;
var b2 = rect2.top + rect2.height;
return (l1 < r2 && r1 > l2 && t1 < b2 && b1 > t2);
}
/**
* @constructor
*/
var SelectToSpeak = function() {
/** @private { AutomationNode } */
this.node_ = null;
/** @private { boolean } */
this.down_ = false;
/** @private {{x: number, y: number}} */
this.mouseStart_ = {x: 0, y: 0};
chrome.automation.getDesktop(function(desktop) {
desktop.addEventListener(
EventType.mousePressed, this.onMousePressed_.bind(this), true);
desktop.addEventListener(
EventType.mouseDragged, this.onMouseDragged_.bind(this), true);
desktop.addEventListener(
EventType.mouseReleased, this.onMouseReleased_.bind(this), true);
desktop.addEventListener(
EventType.mouseCanceled, this.onMouseCanceled_.bind(this), true);
}.bind(this));
};
SelectToSpeak.prototype = {
/**
* Called when the mouse is pressed and the user is in a mode where
* select-to-speak is capturing mouse events (for example holding down
* Search).
*
* @param {!AutomationEvent} evt
*/
onMousePressed_: function(evt) {
this.down_ = true;
this.mouseStart_ = {x: evt.mouseX, y: evt.mouseY};
this.startNode_ = evt.target;
chrome.tts.stop();
this.onMouseDragged_(evt);
},
/**
* Called when the mouse is moved or dragged and the user is in a
* mode where select-to-speak is capturing mouse events (for example
* holding down Search).
*
* @param {!AutomationEvent} evt
*/
onMouseDragged_: function(evt) {
if (!this.down_)
return;
var rect = rectFromPoints(
this.mouseStart_.x, this.mouseStart_.y,
evt.mouseX, evt.mouseY);
chrome.accessibilityPrivate.setFocusRing([rect]);
},
/**
* Called when the mouse is released and the user is in a
* mode where select-to-speak is capturing mouse events (for example
* holding down Search).
*
* @param {!AutomationEvent} evt
*/
onMouseReleased_: function(evt) {
this.onMouseDragged_(evt);
this.down_ = false;
chrome.accessibilityPrivate.setFocusRing([]);
// Walk up to the nearest window, web area, or dialog that the
// hit node is contained inside. Only speak objects within that
// container. In the future we might include other container-like
// roles here.
var root = this.startNode_;
while (root.parent &&
root.role != RoleType.window &&
root.role != RoleType.rootWebArea &&
root.role != RoleType.desktop &&
root.role != RoleType.dialog) {
root = root.parent;
}
var rect = rectFromPoints(
this.mouseStart_.x, this.mouseStart_.y,
evt.mouseX, evt.mouseY);
var nodes = [];
this.findAllMatching_(root, rect, nodes);
this.startSpeechQueue_(nodes);
},
/**
* Called when the user cancels select-to-speak's capturing of mouse
* events (for example by releasing Search while the mouse is still down).
*
* @param {!AutomationEvent} evt
*/
onMouseCanceled_: function(evt) {
this.down_ = false;
chrome.accessibilityPrivate.setFocusRing([]);
chrome.tts.stop();
},
/**
* Finds all nodes within the subtree rooted at |node| that overlap
* a given rectangle.
* @param {AutomationNode} node The starting node.
* @param {{left: number, top: number, width: number, height: number}} rect
* The bounding box to search.
* @param {Array<AutomationNode>} nodes The matching node array to be
* populated.
* @return {boolean} True if any matches are found.
*/
findAllMatching_: function(node, rect, nodes) {
var found = false;
for (var c = node.firstChild; c; c = c.nextSibling) {
if (this.findAllMatching_(c, rect, nodes))
found = true;
}
if (found)
return true;
if (!node.name || !node.location)
return false;
if (overlaps(node.location, rect)) {
nodes.push(node);
return true;
}
return false;
},
/**
* Enqueue speech commands for all of the given nodes.
* @param {Array<AutomationNode>} nodes The nodes to speak.
*/
startSpeechQueue_: function(nodes) {
chrome.tts.stop();
for (var i = 0; i < nodes.length; i++) {
var node = nodes[i];
var isLast = (i == nodes.length - 1);
chrome.tts.speak(node.name, {
lang: 'en-US',
'enqueue': true,
onEvent: (function(node, isLast, event) {
if (event.type == 'start') {
chrome.accessibilityPrivate.setFocusRing([node.location]);
} else if (event.type == 'interrupted' ||
event.type == 'cancelled') {
chrome.accessibilityPrivate.setFocusRing([]);
} else if (event.type == 'end') {
if (isLast) {
chrome.accessibilityPrivate.setFocusRing([]);
}
}
}).bind(this, node, isLast)
});
}
}
};
new SelectToSpeak();
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/**
* Test fixture for select_to_speak.js.
* @constructor
* @extends {testing.Test}
*/
function SelectToSpeakUnitTest () {
testing.Test.call(this);
}
SelectToSpeakUnitTest.prototype = {
__proto__: testing.Test.prototype,
/** @override */
extraLibraries: [
'test_support.js',
'select_to_speak.js'
]
};
TEST_F('SelectToSpeakUnitTest', 'Overlaps', function() {
var rect1 = {left: 0, top: 0, width: 100, height: 100};
var rect2 = {left: 80, top: 0, width: 100, height: 20};
var rect3 = {left: 0, top: 80, width: 20, height: 100};
assertTrue(overlaps(rect1, rect1));
assertTrue(overlaps(rect2, rect2));
assertTrue(overlaps(rect3, rect3));
assertTrue(overlaps(rect1, rect2));
assertTrue(overlaps(rect1, rect3));
assertFalse(overlaps(rect2, rect3));
});
TEST_F('SelectToSpeakUnitTest', 'RectFromPoints', function() {
var rect = {left: 10, top: 20, width: 50, height: 60};
assertNotEquals(
JSON.stringify(rect),
JSON.stringify(rectFromPoints(0, 0, 10, 10)));
assertEquals(
JSON.stringify(rect),
JSON.stringify(rectFromPoints(10, 20, 60, 80)));
assertEquals(
JSON.stringify(rect),
JSON.stringify(rectFromPoints(60, 20, 10, 80)));
assertEquals(
JSON.stringify(rect),
JSON.stringify(rectFromPoints(10, 80, 60, 20)));
assertEquals(
JSON.stringify(rect),
JSON.stringify(rectFromPoints(60, 80, 10, 20)));
});
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/**
* Stubs out extension API functions so that SelectToSpeakUnitTest
* can load.
*/
chrome.automation = {};
/**
* Stub
*/
chrome.automation.getDesktop = function() {};
......@@ -268,7 +268,7 @@
// Arguments for the find() and findAll() methods.
[nocompile, noinline_doc] dictionary FindParams {
automation.RoleType? role;
RoleType? role;
// A map of $(ref:automation.StateType) to boolean, indicating for each
// state whether it should be set or not. For example:
......
......@@ -3017,9 +3017,15 @@ js2gtest("unit_tests_js") {
"//ui/webui/resources/js/cr.js",
]
if (is_chromeos) {
sources += [ "../browser/resources/chromeos/braille_ime/braille_ime_unittest.gtestjs" ]
extra_js_files +=
[ "../browser/resources/chromeos/braille_ime/braille_ime.js" ]
sources += [
"../browser/resources/chromeos/braille_ime/braille_ime_unittest.gtestjs",
"../browser/resources/chromeos/select_to_speak/select_to_speak_unittest.gtestjs",
]
extra_js_files += [
"../browser/resources/chromeos/braille_ime/braille_ime.js",
"../browser/resources/chromeos/select_to_speak/select_to_speak.js",
"../browser/resources/chromeos/select_to_speak/test_support.js",
]
}
}
......
......@@ -18,6 +18,7 @@
'<(DEPTH)/chrome/browser/resources/chromeos/login/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/chromeos/network_ui/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/chromeos/quick_unlock/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/chromeos/select_to_speak/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/extensions/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/history/compiled_resources2.gyp:*',
'<(DEPTH)/chrome/browser/resources/md_downloads/compiled_resources2.gyp:*',
......
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This file was generated by:
// tools/json_schema_compiler/compiler.py.
// NOTE: The format of types has changed. 'FooType' is now
// 'chrome.accessibilityPrivate.FooType'.
// Please run the closure compiler before committing changes.
// See https://chromium.googlesource.com/chromium/src/+/master/docs/closure_compilation.md
/** @fileoverview Externs generated from namespace: accessibilityPrivate */
/**
* @const
*/
chrome.accessibilityPrivate = {};
/**
* Information about an alert
* @typedef {{
* message: string
* }}
* @see https://developer.chrome.com/extensions/accessibilityPrivate#type-AlertInfo
*/
chrome.accessibilityPrivate.AlertInfo;
/**
* Bounding rectangle in global screen coordinates.
* @typedef {{
* left: number,
* top: number,
* width: number,
* height: number
* }}
* @see https://developer.chrome.com/extensions/accessibilityPrivate#type-ScreenRect
*/
chrome.accessibilityPrivate.ScreenRect;
/**
* @enum {string}
* @see https://developer.chrome.com/extensions/accessibilityPrivate#type-Gesture
*/
chrome.accessibilityPrivate.Gesture = {
CLICK: 'click',
SWIPE_LEFT1: 'swipeLeft1',
SWIPE_UP1: 'swipeUp1',
SWIPE_RIGHT1: 'swipeRight1',
SWIPE_DOWN1: 'swipeDown1',
SWIPE_LEFT2: 'swipeLeft2',
SWIPE_UP2: 'swipeUp2',
SWIPE_RIGHT2: 'swipeRight2',
SWIPE_DOWN2: 'swipeDown2',
SWIPE_LEFT3: 'swipeLeft3',
SWIPE_UP3: 'swipeUp3',
SWIPE_RIGHT3: 'swipeRight3',
SWIPE_DOWN3: 'swipeDown3',
SWIPE_LEFT4: 'swipeLeft4',
SWIPE_UP4: 'swipeUp4',
SWIPE_RIGHT4: 'swipeRight4',
SWIPE_DOWN4: 'swipeDown4',
};
/**
* Enables or disables native accessibility support. Once disabled, it is up to
* the calling extension to provide accessibility for web contents.
* @param {boolean} enabled True if native accessibility support should be
* enabled.
* @see https://developer.chrome.com/extensions/accessibilityPrivate#method-setNativeAccessibilityEnabled
*/
chrome.accessibilityPrivate.setNativeAccessibilityEnabled = function(enabled) {};
/**
* Set the bounds of the accessibility focus ring.
* @param {!Array<!chrome.accessibilityPrivate.ScreenRect>} rects Array of
* rectangles to draw the accessibility focus ring around.
* @see https://developer.chrome.com/extensions/accessibilityPrivate#method-setFocusRing
*/
chrome.accessibilityPrivate.setFocusRing = function(rects) {};
/**
* Sets the calling extension as a listener of all keyboard events optionally
* allowing the calling extension to capture/swallow the key event via DOM apis.
* Returns false via callback when unable to set the listener.
* @param {boolean} enabled True if the caller wants to listen to key events;
* false to stop listening to events. Note that there is only ever one
* extension listening to key events.
* @param {boolean} capture True if key events should be swallowed natively and
* not propagated if preventDefault() gets called by the extension's
* background page.
* @see https://developer.chrome.com/extensions/accessibilityPrivate#method-setKeyboardListener
*/
chrome.accessibilityPrivate.setKeyboardListener = function(enabled, capture) {};
/**
* Fired whenever ChromeVox should output introduction.
* @type {!ChromeEvent}
* @see https://developer.chrome.com/extensions/accessibilityPrivate#event-onIntroduceChromeVox
*/
chrome.accessibilityPrivate.onIntroduceChromeVox;
/**
* Fired when an accessibility gesture is detected by the touch exploration
* controller.
* @type {!ChromeEvent}
* @see https://developer.chrome.com/extensions/accessibilityPrivate#event-onAccessibilityGesture
*/
chrome.accessibilityPrivate.onAccessibilityGesture;
This diff is collapsed.
......@@ -8,10 +8,18 @@
########################################################
{
'targets': [
{
'target_name': 'accessibility_private',
'includes': ['../include_js.gypi'],
},
{
'target_name': 'autofill_private',
'includes': ['../include_js.gypi'],
},
{
'target_name': 'automation',
'includes': ['../include_js.gypi'],
},
{
'target_name': 'bluetooth',
'includes': ['../include_js.gypi'],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment