Implement ITextProvider and ITextRangeProvider for UIA (flutter#38538)

* Import files from Chromium as-is * Modify AX to compile * Disable new untitests for now * Formatting * License * License * License * License * Excl files * Update third_party/accessibility/ax/ax_node.h Co-authored-by: Loïc Sharma <[email protected]> * Update third_party/accessibility/ax/ax_node.h Co-authored-by: Loïc Sharma <[email protected]> * Typo * Update third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win.h Co-authored-by: Loïc Sharma <[email protected]> * Add clipping_behavior, tree_id * Formatting * Fix bad patch * TODOs Co-authored-by: Loïc Sharma <[email protected]>
loic-sharma · Jan 3, 2023 · d85adb6 · d85adb6
1 parent a4ce69f
commit d85adb6
Show file tree

Hide file tree

Showing 21 changed files with 11,203 additions and 14 deletions.
diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files
@@ -358,6 +358,8 @@
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_base_unittest.cc
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_mac_unittest.h
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_mac_unittest.mm
+../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textprovider_win_unittest.cc
+../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win_unittest.cc
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_unittest.cc
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_unittest.h
 ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_win_unittest.cc

diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter
@@ -6072,6 +6072,11 @@ ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_fragment_root_
 ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_fragment_root_win.h + ../../../flutter/third_party/accessibility/LICENSE
 ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_utils_win.cc + ../../../flutter/third_party/accessibility/LICENSE
 ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_utils_win.h + ../../../flutter/third_party/accessibility/LICENSE
+ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textprovider_win.cc + ../../../flutter/third_party/accessibility/LICENSE
+ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textprovider_win.h + ../../../flutter/third_party/accessibility/LICENSE
+ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win.cc + ../../../flutter/third_party/accessibility/LICENSE
+ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win.h + ../../../flutter/third_party/accessibility/LICENSE
+ORIGIN: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_tree_manager.h + ../../../flutter/third_party/accessibility/LICENSE
 ORIGIN: ../../../flutter/third_party/accessibility/base/win/scoped_safearray.h + ../../../flutter/third_party/accessibility/LICENSE
 TYPE: LicenseType.bsd
 FILE: ../../../flutter/third_party/accessibility/ax/ax_active_popup.cc
@@ -6091,6 +6096,11 @@ FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_fragment_root_wi
 FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_fragment_root_win.h
 FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_utils_win.cc
 FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_utils_win.h
+FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textprovider_win.cc
+FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textprovider_win.h
+FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win.cc
+FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_node_textrangeprovider_win.h
+FILE: ../../../flutter/third_party/accessibility/ax/platform/ax_platform_tree_manager.h
 FILE: ../../../flutter/third_party/accessibility/base/win/scoped_safearray.h
 ----------------------------------------------------------------------------------------------------
 Copyright 2019 The Chromium Authors. All rights reserved.

diff --git a/third_party/accessibility/BUILD.gn b/third_party/accessibility/BUILD.gn
@@ -91,6 +91,8 @@ if (enable_unittests) {
       if (is_win) {
         sources += [
           "ax/platform/ax_fragment_root_win_unittest.cc",
+          "ax/platform/ax_platform_node_textprovider_win_unittest.cc",
+          "ax/platform/ax_platform_node_textrangeprovider_win_unittest.cc",
           "ax/platform/ax_platform_node_win_unittest.cc",
           "base/win/dispatch_stub.cc",
           "base/win/dispatch_stub.h",

diff --git a/third_party/accessibility/ax/BUILD.gn b/third_party/accessibility/ax/BUILD.gn
@@ -17,6 +17,7 @@ source_set("ax") {
     "platform/ax_platform_node_delegate.h",
     "platform/ax_platform_node_delegate_base.cc",
     "platform/ax_platform_node_delegate_base.h",
+    "platform/ax_platform_tree_manager.h",
     "platform/ax_unique_id.cc",
     "platform/ax_unique_id.h",
     "platform/compute_attributes.cc",
@@ -92,6 +93,10 @@ source_set("ax") {
       "platform/ax_fragment_root_win.h",
       "platform/ax_platform_node_delegate_utils_win.cc",
       "platform/ax_platform_node_delegate_utils_win.h",
+      "platform/ax_platform_node_textprovider_win.cc",
+      "platform/ax_platform_node_textprovider_win.h",
+      "platform/ax_platform_node_textrangeprovider_win.cc",
+      "platform/ax_platform_node_textrangeprovider_win.h",
       "platform/ax_platform_node_win.cc",
       "platform/ax_platform_node_win.h",
       "platform/uia_registrar_win.cc",

diff --git a/third_party/accessibility/ax/ax_node.cc b/third_party/accessibility/ax/ax_node.cc
@@ -11,6 +11,8 @@
 #include "ax_role_properties.h"
 #include "ax_table_info.h"
 #include "ax_tree.h"
+#include "ax_tree_manager.h"
+#include "ax_tree_manager_map.h"
 #include "base/color_utils.h"
 #include "base/string_utils.h"
 
@@ -1197,6 +1199,30 @@ bool AXNode::IsEmbeddedGroup() const {
   return ui::IsSetLike(parent()->data().role);
 }
 
+AXNode* AXNode::GetLowestPlatformAncestor() const {
+  AXNode* current_node = const_cast<AXNode*>(this);
+  AXNode* lowest_unignored_node = current_node;
+  for (; lowest_unignored_node && lowest_unignored_node->IsIgnored();
+       lowest_unignored_node = lowest_unignored_node->parent()) {
+  }
+
+  // `highest_leaf_node` could be nullptr.
+  AXNode* highest_leaf_node = lowest_unignored_node;
+  // For the purposes of this method, a leaf node does not include leaves in the
+  // internal accessibility tree, only in the platform exposed tree.
+  for (AXNode* ancestor_node = lowest_unignored_node; ancestor_node;
+       ancestor_node = ancestor_node->GetUnignoredParent()) {
+    if (ancestor_node->IsLeaf())
+      highest_leaf_node = ancestor_node;
+  }
+  if (highest_leaf_node)
+    return highest_leaf_node;
+
+  if (lowest_unignored_node)
+    return lowest_unignored_node;
+  return current_node;
+}
+
 AXNode* AXNode::GetTextFieldAncestor() const {
   AXNode* parent = GetUnignoredParent();
 
@@ -1210,4 +1236,69 @@ AXNode* AXNode::GetTextFieldAncestor() const {
   return nullptr;
 }
 
+bool AXNode::IsDescendantOfCrossingTreeBoundary(const AXNode* ancestor) const {
+  if (!ancestor)
+    return false;
+  if (this == ancestor)
+    return true;
+  if (const AXNode* parent = GetParentCrossingTreeBoundary())
+    return parent->IsDescendantOfCrossingTreeBoundary(ancestor);
+  return false;
+}
+
+AXNode* AXNode::GetParentCrossingTreeBoundary() const {
+  BASE_DCHECK(!tree_->GetTreeUpdateInProgressState());
+  if (parent_)
+    return parent_;
+  const AXTreeManager* manager =
+      AXTreeManagerMap::GetInstance().GetManager(tree_->GetAXTreeID());
+  if (manager)
+    return manager->GetParentNodeFromParentTreeAsAXNode();
+  return nullptr;
+}
+
+AXTree::Selection AXNode::GetUnignoredSelection() const {
+  BASE_DCHECK(tree())
+      << "Cannot retrieve the current selection if the node is not "
+         "attached to an accessibility tree.\n"
+      << *this;
+  AXTree::Selection selection = tree()->GetUnignoredSelection();
+
+  // "selection.anchor_offset" and "selection.focus_ofset" might need to be
+  // adjusted if the anchor or the focus nodes include ignored children.
+  //
+  // TODO(nektar): Move this logic into its own "AXSelection" class and cache
+  // the result for faster reuse.
+  const AXNode* anchor = tree()->GetFromId(selection.anchor_object_id);
+  if (anchor && !anchor->IsLeaf()) {
+    BASE_DCHECK(selection.anchor_offset >= 0);
+    if (static_cast<size_t>(selection.anchor_offset) <
+        anchor->children().size()) {
+      const AXNode* anchor_child = anchor->children()[selection.anchor_offset];
+      BASE_DCHECK(anchor_child);
+      selection.anchor_offset =
+          static_cast<int>(anchor_child->GetUnignoredIndexInParent());
+    } else {
+      selection.anchor_offset =
+          static_cast<int>(anchor->GetUnignoredChildCount());
+    }
+  }
+
+  const AXNode* focus = tree()->GetFromId(selection.focus_object_id);
+  if (focus && !focus->IsLeaf()) {
+    BASE_DCHECK(selection.focus_offset >= 0);
+    if (static_cast<size_t>(selection.focus_offset) <
+        focus->children().size()) {
+      const AXNode* focus_child = focus->children()[selection.focus_offset];
+      BASE_DCHECK(focus_child);
+      selection.focus_offset =
+          static_cast<int>(focus_child->GetUnignoredIndexInParent());
+    } else {
+      selection.focus_offset =
+          static_cast<int>(focus->GetUnignoredChildCount());
+    }
+  }
+  return selection;
+}
+
 }  // namespace ui
diff --git a/third_party/accessibility/ax/ax_node.h b/third_party/accessibility/ax/ax_node.h
@@ -120,6 +120,10 @@ class AX_EXPORT AXNode final {
   size_t GetUnignoredChildCount() const;
   AXNode* GetUnignoredChildAtIndex(size_t index) const;
   AXNode* GetUnignoredParent() const;
+  // Gets the unignored selection from the accessibility tree, meaning the
+  // selection whose endpoints are on unignored nodes. (An "ignored" node is a
+  // node that is not exposed to platform APIs: See `IsIgnored`.)
+  OwnerTree::Selection GetUnignoredSelection() const;
   size_t GetUnignoredIndexInParent() const;
   size_t GetIndexInParent() const;
   AXNode* GetFirstUnignoredChild() const;
@@ -191,6 +195,9 @@ class AX_EXPORT AXNode final {
   // Return true if this object is equal to or a descendant of |ancestor|.
   bool IsDescendantOf(const AXNode* ancestor) const;
 
+  bool IsDescendantOfCrossingTreeBoundary(const AXNode* ancestor) const;
+  AXNode* GetParentCrossingTreeBoundary() const;
+
   // Gets the text offsets where new lines start either from the node's data or
   // by computing them and caching the result.
   std::vector<int> GetOrComputeLineStartOffsets();
@@ -436,6 +443,12 @@ class AX_EXPORT AXNode final {
   // Finds and returns a pointer to ordered set containing node.
   AXNode* GetOrderedSet() const;
 
+  // If this node is exposed to the platform's accessibility layer, returns this
+  // node. Otherwise, returns the lowest ancestor that is exposed to the
+  // platform. (See `IsLeaf` and `IsIgnored` for information on what is
+  // exposed to platform APIs.)
+  AXNode* GetLowestPlatformAncestor() const;
+
  private:
   // Computes the text offset where each line starts by traversing all child
   // leaf nodes.

diff --git a/third_party/accessibility/ax/ax_node_position.cc b/third_party/accessibility/ax/ax_node_position.cc
@@ -20,6 +20,16 @@ AXEmbeddedObjectBehavior g_ax_embedded_object_behavior =
     AXEmbeddedObjectBehavior::kSuppressCharacter;
 #endif  // defined(OS_WIN)
 
+ScopedAXEmbeddedObjectBehaviorSetter::ScopedAXEmbeddedObjectBehaviorSetter(
+    AXEmbeddedObjectBehavior behavior) {
+  prev_behavior_ = g_ax_embedded_object_behavior;
+  g_ax_embedded_object_behavior = behavior;
+}
+
+ScopedAXEmbeddedObjectBehaviorSetter::~ScopedAXEmbeddedObjectBehaviorSetter() {
+  g_ax_embedded_object_behavior = prev_behavior_;
+}
+
 // static
 AXNodePosition::AXPositionInstance AXNodePosition::CreatePosition(
     const AXNode& node,
@@ -205,8 +215,9 @@ std::u16string AXNodePosition::GetText() const {
         ax::mojom::StringAttribute::kName);
   }
 
-  for (int i = 0; i < AnchorChildCount(); ++i)
+  for (int i = 0; i < AnchorChildCount(); ++i) {
     text += CreateChildPositionAt(i)->GetText();
+  }
 
   return text;
 }
@@ -263,8 +274,9 @@ int AXNodePosition::MaxTextOffset() const {
   }
 
   int text_length = 0;
-  for (int i = 0; i < AnchorChildCount(); ++i)
+  for (int i = 0; i < AnchorChildCount(); ++i) {
     text_length += CreateChildPositionAt(i)->MaxTextOffset();
+  }
 
   return text_length;
 }
@@ -286,9 +298,10 @@ bool AXNodePosition::IsInLineBreakingObject() const {
   if (IsNullPosition())
     return false;
   BASE_DCHECK(GetAnchor());
-  return GetAnchor()->data().GetBoolAttribute(
-             ax::mojom::BoolAttribute::kIsLineBreakingObject) &&
-         !GetAnchor()->IsInListMarker();
+  return (GetAnchor()->data().GetBoolAttribute(
+              ax::mojom::BoolAttribute::kIsLineBreakingObject) &&
+          !GetAnchor()->IsInListMarker()) ||
+         GetAnchor()->data().role == ax::mojom::Role::kLineBreak;
 }
 
 ax::mojom::Role AXNodePosition::GetAnchorRole() const {

diff --git a/third_party/accessibility/ax/ax_position.h b/third_party/accessibility/ax/ax_position.h
@@ -105,6 +105,16 @@ enum class AXEmbeddedObjectBehavior {
 // overridden for testing.
 AX_EXPORT extern AXEmbeddedObjectBehavior g_ax_embedded_object_behavior;
 
+class AX_EXPORT ScopedAXEmbeddedObjectBehaviorSetter {
+ public:
+  explicit ScopedAXEmbeddedObjectBehaviorSetter(
+      AXEmbeddedObjectBehavior behavior);
+  ~ScopedAXEmbeddedObjectBehaviorSetter();
+
+ private:
+  AXEmbeddedObjectBehavior prev_behavior_;
+};
+
 // Forward declarations.
 template <class AXPositionType, class AXNodeType>
 class AXPosition;
@@ -324,8 +334,9 @@ class AXPosition {
     BASE_DCHECK(GetAnchor());
     // If this position is anchored to an ignored node, then consider this
     // position to be ignored.
-    if (GetAnchor()->IsIgnored())
+    if (GetAnchor()->IsIgnored()) {
       return true;
+    }
 
     switch (kind_) {
       case AXPositionKind::NULL_POSITION:
@@ -372,8 +383,9 @@ class AXPosition {
         // If the corresponding leaf position is ignored, the current text
         // offset will point to ignored text. Therefore, consider this position
         // to be ignored.
-        if (!IsLeaf())
+        if (!IsLeaf()) {
           return AsLeafTreePosition()->IsIgnored();
+        }
         return false;
     }
   }
@@ -417,8 +429,9 @@ class AXPosition {
                 (child_index_ >= 0 && child_index_ <= AnchorChildCount())) &&
                !IsInDescendantOfEmptyObject();
       case AXPositionKind::TEXT_POSITION:
-        if (!GetAnchor() || IsInDescendantOfEmptyObject())
+        if (!GetAnchor() || IsInDescendantOfEmptyObject()) {
           return false;
+        }
 
         // For performance reasons we skip any validation of the text offset
         // that involves retrieving the anchor's text, if the offset is set to
@@ -1029,8 +1042,9 @@ class AXPosition {
       const AXNodeType* ancestor_anchor,
       ax::mojom::MoveDirection move_direction =
           ax::mojom::MoveDirection::kForward) const {
-    if (!ancestor_anchor)
+    if (!ancestor_anchor) {
       return CreateNullPosition();
+    }
 
     AXPositionInstance ancestor_position = Clone();
     while (!ancestor_position->IsNullPosition() &&
@@ -1285,8 +1299,9 @@ class AXPosition {
   }
 
   AXPositionInstance AsLeafTextPosition() const {
-    if (IsNullPosition() || IsLeaf())
+    if (IsNullPosition() || IsLeaf()) {
       return AsTextPosition();
+    }
 
     // Adjust the text offset.
     // No need to check for "before text" positions here because they are only
@@ -1316,7 +1331,7 @@ class AXPosition {
           child_position->affinity_ = ax::mojom::TextAffinity::kUpstream;
           break;
         }
-        child_position = text_position->CreateChildPositionAt(i);
+        child_position = std::move(text_position->CreateChildPositionAt(i));
         adjusted_offset -= max_text_offset_in_parent;
       }
 
@@ -1902,7 +1917,7 @@ class AXPosition {
         // the same as the one that would have been computed if the original
         // position were at the start of the inline text box for "Line two".
         const int max_text_offset = MaxTextOffset();
-        const int max_text_offset_in_parent =
+        int max_text_offset_in_parent =
             IsEmbeddedObjectInParent() ? 1 : max_text_offset;
         int parent_offset = AnchorTextOffsetInParent();
         ax::mojom::TextAffinity parent_affinity = affinity_;
@@ -1935,6 +1950,14 @@ class AXPosition {
           parent_affinity = ax::mojom::TextAffinity::kDownstream;
         }
 
+        // This dummy position serves to retrieve the max text offset of the
+        // anchor-node in which we want to create the parent position.
+        AXPositionInstance dummy_position =
+            CreateTextPosition(tree_id, parent_id, 0, parent_affinity);
+        max_text_offset_in_parent = dummy_position->MaxTextOffset();
+        if (parent_offset > max_text_offset_in_parent) {
+          parent_offset = max_text_offset_in_parent;
+        }
         AXPositionInstance parent_position = CreateTextPosition(
             tree_id, parent_id, parent_offset, parent_affinity);
 
@@ -2061,6 +2084,7 @@ class AXPosition {
       BASE_DCHECK(text_position->text_offset_ >= 0);
       return text_position;
     }
+
     text_position = text_position->CreateNextLeafTextPosition();
     while (!text_position->IsNullPosition() &&
            (text_position->IsIgnored() || !text_position->MaxTextOffset())) {

diff --git a/third_party/accessibility/ax/ax_range.h b/third_party/accessibility/ax/ax_range.h
@@ -11,6 +11,7 @@
 #include <utility>
 #include <vector>
 
+#include "ax_clipping_behavior.h"
 #include "ax_enums.h"
 #include "ax_offscreen_result.h"
 #include "ax_role_properties.h"
@@ -35,6 +36,7 @@ class AXRangeRectDelegate {
       AXNode::AXID node_id,
       int start_offset,
       int end_offset,
+      ui::AXClippingBehavior clipping_behavior,
       AXOffscreenResult* offscreen_result) = 0;
   virtual gfx::Rect GetBoundsRect(AXTreeID tree_id,
                                   AXNode::AXID node_id,
@@ -392,7 +394,8 @@ class AXRange {
                     current_line_start->tree_id(),
                     current_line_start->anchor_id(),
                     current_line_start->text_offset(),
-                    current_line_end->text_offset(), &offscreen_result)
+                    current_line_end->text_offset(),
+                    ui::AXClippingBehavior::kUnclipped, &offscreen_result)
               : delegate->GetBoundsRect(current_line_start->tree_id(),
                                         current_line_start->anchor_id(),
                                         &offscreen_result);