pyAutoGUI (#30)

* add pyautogui support
* add GetBoundingBox command
* add example pyautogui script
* travis ci: use focal dist for linux, replace g++5 with g++10

.travis.yml

@@ -1,15 +1,15 @@
 language: cpp
 sudo: required
-dist: xenial
+dist: focal
 
 matrix:
   include:
     - os: linux
       compiler: gcc
-      addons: &linux-gcc-5 { apt: {
+      addons: &linux-gcc-10 { apt: {
         sources: [ ubuntu-toolchain-r-test ],
-        packages: [ g++-5, cmake, qt5-default, qtdeclarative5-dev ] } }
-      env: MATRIX_EVAL="CXX=g++-5"  BUILD_TYPE=Release
+        packages: [ g++-10, cmake, qt5-default, qtdeclarative5-dev ] } }
+      env: MATRIX_EVAL="CXX=g++-10"  BUILD_TYPE=Release
 
     - os: linux
       compiler: gcc

README.md

@@ -33,12 +33,33 @@ resultText = s.getStringProperty("mainWindow/results", "text")
 s.quit()
 ```
 
+You can also use [PyAutoGUI](https://pyautogui.readthedocs.io) in combination with
+Spix. Have a look at the [example script](examples/RemoteCtrl/script/autogui.py).
+
 ## What are the applications of Spix?
-The obvious use for Spix is to automatically test the GUI of your Qt/QML application
+The main use for Spix is to automatically test the UI of your Qt/QML application
 and make sure that it behaves as you expect. However, you can also use Spix as
 an easy way to remote control existing Qt/QML applications or to automatically
 generate and update screenshots for your documentation.
 
+## Two modes of operation
+Spix can be used in two ways, which are different in how events are generated and sent
+to your application:
+
+### Generate Qt events directly
+You can use Spix to directly create Qt events, either from C++ as a unit test, or from
+an external script via the network through RPC. Since the Qt events are generated directly inside the
+app, and do not come from the system, the mouse coursor will not actually move and interaction
+with other applications is limited. On the plus side, this mechanism is independent from
+the system your app is running on and can easily be used to control software on an embedded
+device via the network (RPC).
+
+### Generate system events externally
+In this case, Spix is not generating the events itself. Instead, you use a script to query
+Spix for the screen coordinates of qt objects and then generate events on the system level
+through other tools. One option is to use python together with PyAutoGUI for this, as is
+done in the [RemoteCtrl](examples/RemoteCtrl) example.
+
 # Requirements
 * Qt
 * AnyRPC

examples/RemoteCtrl/script/autogui.py

@@ -0,0 +1,36 @@
+import xmlrpc.client
+import time
+import pyautogui # you might have to first `pip install pyautogui`
+
+
+def clickItem(path):
+    # Query spix to get the location of the item
+    bounds = s.getBoundingBox(path)
+    # bounds = [x, y, width, height] as list
+    center_x = bounds[0] + bounds[2] / 2
+    center_y = bounds[1] + bounds[3] / 2
+
+    # use pyautogui to move the cursor...
+    pyautogui.moveTo(center_x, center_y, duration=1)
+    # ...and click
+    pyautogui.click()
+
+
+
+# Connect to the rpc server running in the qt app
+s = xmlrpc.client.ServerProxy('http://localhost:9000')
+
+# Wait a bit so that the user has time to move the app to the foreground
+print("Make sure the qt app is in the foreground... waiting 3s...")
+time.sleep(3)
+
+# Click buttons
+clickItem("mainWindow/Button_1")
+clickItem("mainWindow/Button_2")
+clickItem("mainWindow/Button_2")
+clickItem("mainWindow/Button_1")
+clickItem("mainWindow/Button_2")
+
+# Query the qt app for the text contents of the results box
+resultText = s.getStringProperty("mainWindow/results", "text")
+print("Result:\n{}".format(resultText))

examples/RemoteCtrl/script/autogui_requirements.txt

@@ -0,0 +1 @@
+PyAutoGUI==0.9.52

lib/CMakeLists.txt

@@ -47,6 +47,8 @@ set(SOURCES
     src/Commands/EnterKey.h
     src/Commands/ExistsAndVisible.cpp
     src/Commands/ExistsAndVisible.h
+    src/Commands/GetBoundingBox.cpp
+    src/Commands/GetBoundingBox.h
     src/Commands/GetProperty.cpp
     src/Commands/GetProperty.h
     src/Commands/GetTestStatus.cpp

lib/include/Spix/Data/Geometry.h

@@ -11,17 +11,27 @@ namespace spix {
 using Real = double;
 
 struct Size {
+    Size();
     Size(Real width, Real height);
 
     Real width;
     Real height;
 };
 
 struct Point {
+    Point();
     Point(Real x, Real y);
 
     Real x;
     Real y;
 };
 
+struct Rect {
+    Rect();
+    Rect(Real x, Real y, Real width, Real height);
+
+    Point topLeft;
+    Size size;
+};
+
 } // namespace spix

lib/include/Spix/TestServer.h

@@ -11,6 +11,7 @@
 #include <memory>
 #include <thread>
 
+#include <Spix/Data/Geometry.h>
 #include <Spix/Data/ItemPath.h>
 
 namespace spix {
@@ -52,6 +53,7 @@ class TestServer {
 
     std::string getStringProperty(ItemPath path, std::string propertyName);
     void setStringProperty(ItemPath path, std::string propertyName, std::string propertyValue);
+    Rect getBoundingBox(ItemPath path);
     bool existsAndVisible(ItemPath path);
     std::vector<std::string> getErrors();
 

lib/src/AnyRpcServer.cpp

@@ -51,6 +51,11 @@ AnyRpcServer::AnyRpcServer(int anyrpcPort)
         [this](std::string path, std::string property, std::string value) {
             setStringProperty(std::move(path), std::move(property), std::move(value));
         });
+    utils::AddFunctionToAnyRpc<std::vector<double>(std::string)>(methodManager, "getBoundingBox",
+        "Return the bounding box of an item in screen coordinates", [this](std::string path) {
+            auto bounds = getBoundingBox(std::move(path));
+            return std::vector<double> {bounds.topLeft.x, bounds.topLeft.y, bounds.size.width, bounds.size.height};
+        });
     utils::AddFunctionToAnyRpc<bool(std::string)>(methodManager, "existsAndVisible",
         "Returns true if the given object exists",
         [this](std::string path) { return existsAndVisible(std::move(path)); });

lib/src/Commands/GetBoundingBox.cpp

@@ -0,0 +1,34 @@
+/***
+ * Copyright (C) Falko Axmann. All rights reserved.
+ * Licensed under the MIT license.
+ * See LICENSE.txt file in the project root for full license information.
+ ****/
+
+#include "GetBoundingBox.h"
+
+#include <Scene/Scene.h>
+
+namespace spix {
+namespace cmd {
+
+GetBoundingBox::GetBoundingBox(ItemPath path, std::promise<Rect> promise)
+: m_path(std::move(path))
+, m_promise(std::move(promise))
+{
+}
+
+void GetBoundingBox::execute(CommandEnvironment& env)
+{
+    auto item = env.scene().itemAtPath(m_path);
+
+    if (item) {
+        auto value = item->bounds();
+        m_promise.set_value(value);
+    } else {
+        m_promise.set_value(Rect {0.0, 0.0, 0.0, 0.0});
+        env.state().reportError("GetBoundingBox: Item not found: " + m_path.string());
+    }
+}
+
+} // namespace cmd
+} // namespace spix

lib/src/Commands/GetBoundingBox.h

@@ -0,0 +1,30 @@
+/***
+ * Copyright (C) Falko Axmann. All rights reserved.
+ * Licensed under the MIT license.
+ * See LICENSE.txt file in the project root for full license information.
+ ****/
+
+#pragma once
+
+#include "Command.h"
+#include <Spix/Data/Geometry.h>
+#include <Spix/Data/ItemPath.h>
+
+#include <future>
+
+namespace spix {
+namespace cmd {
+
+class GetBoundingBox : public Command {
+public:
+    GetBoundingBox(ItemPath path, std::promise<Rect> promise);
+
+    void execute(CommandEnvironment& env) override;
+
+private:
+    ItemPath m_path;
+    std::promise<Rect> m_promise;
+};
+
+} // namespace cmd
+} // namespace spix

lib/src/Data/Geometry.cpp

@@ -8,16 +8,38 @@
 
 namespace spix {
 
+Size::Size()
+: width(0.0)
+, height(0.0)
+{
+}
+
 Size::Size(Real width, Real height)
 : width(width)
 , height(height)
 {
 }
 
+Point::Point()
+: x(0.0)
+, y(0.0)
+{
+}
+
 Point::Point(Real x, Real y)
 : x(x)
 , y(y)
 {
 }
 
+Rect::Rect()
+{
+}
+
+Rect::Rect(Real x, Real y, Real width, Real height)
+: topLeft(x, y)
+, size(width, height)
+{
+}
+
 } // namespace spix

lib/src/Scene/Item.h

@@ -25,6 +25,8 @@ class Item {
 
     // Item properties
     virtual Size size() const = 0;
+    virtual Point position() const = 0;
+    virtual Rect bounds() const = 0;
     virtual std::string stringProperty(const std::string& name) const = 0;
     virtual void setStringProperty(const std::string& name, const std::string& value) = 0;
     virtual bool visible() const = 0;

lib/src/Scene/Mock/MockItem.cpp

@@ -18,6 +18,19 @@ Size MockItem::size() const
     return m_size;
 }
 
+Point MockItem::position() const
+{
+    return Point {0.0, 0.0};
+}
+
+Rect MockItem::bounds() const
+{
+    Rect bounds {0.0, 0.0, 0.0, 0.0};
+    bounds.topLeft = position();
+    bounds.size = size();
+    return bounds;
+}
+
 std::string MockItem::stringProperty(const std::string& name) const
 {
     return m_stringProperties.at(name);

lib/src/Scene/Mock/MockItem.h

@@ -18,6 +18,8 @@ class MockItem : public Item {
 
     // Item interface
     Size size() const override;
+    Point position() const override;
+    Rect bounds() const override;
     std::string stringProperty(const std::string& name) const override;
     void setStringProperty(const std::string& name, const std::string& value) override;
     bool visible() const override;

lib/src/Scene/Qt/QtItem.cpp

@@ -20,6 +20,24 @@ Size QtItem::size() const
     return Size {m_item->width(), m_item->height()};
 }
 
+Point QtItem::position() const
+{
+    // the point (0, 0) in item coordinates...
+    QPointF localPoint {0.0, 0.0};
+    // ...is mapped to global to get the item position on screen
+    auto globalPoint = m_item->mapToGlobal(localPoint);
+
+    return Point {globalPoint.rx(), globalPoint.ry()};
+}
+
+Rect QtItem::bounds() const
+{
+    Rect bounds {0.0, 0.0, 0.0, 0.0};
+    bounds.topLeft = position();
+    bounds.size = size();
+    return bounds;
+}
+
 std::string QtItem::stringProperty(const std::string& name) const
 {
     auto value = m_item->property(name.c_str());

lib/src/Scene/Qt/QtItem.h

@@ -18,6 +18,8 @@ class QtItem : public Item {
     QtItem(QQuickItem* item);
 
     Size size() const override;
+    Point position() const override;
+    Rect bounds() const override;
     std::string stringProperty(const std::string& name) const override;
     void setStringProperty(const std::string& name, const std::string& value) override;
     bool visible() const override;

lib/src/TestServer.cpp

@@ -15,6 +15,7 @@
 #include <Commands/DropFromExt.h>
 #include <Commands/EnterKey.h>
 #include <Commands/ExistsAndVisible.h>
+#include <Commands/GetBoundingBox.h>
 #include <Commands/GetProperty.h>
 #include <Commands/GetTestStatus.h>
 #include <Commands/InputText.h>
@@ -111,6 +112,16 @@ void TestServer::setStringProperty(ItemPath path, std::string propertyName, std:
     m_cmdExec->enqueueCommand<cmd::SetProperty>(path, std::move(propertyName), std::move(propertyValue));
 }
 
+Rect TestServer::getBoundingBox(ItemPath path)
+{
+    std::promise<Rect> promise;
+    auto result = promise.get_future();
+    auto cmd = std::make_unique<cmd::GetBoundingBox>(path, std::move(promise));
+    m_cmdExec->enqueueCommand(std::move(cmd));
+
+    return result.get();
+}
+
 bool TestServer::existsAndVisible(ItemPath path)
 {
     std::promise<bool> promise;

lib/src/Utils/AnyRpcUtils.h

@@ -18,6 +18,19 @@
 namespace spix {
 namespace utils {
 
+/**
+ * Generic type traits
+ */
+template <class T>
+struct is_std_vector {
+    static const bool value = false;
+};
+
+template <class T>
+struct is_std_vector<std::vector<T>> {
+    static const bool value = true;
+};
+
 /**
  * Tools to convert an anyrpc::Value to a
  * particular type. If the requested type does not
@@ -79,7 +92,8 @@ std::vector<std::string> unpackAnyRpcParam(anyrpc::Value& value)
  * value to the given anyrpc::Value. If the return type
  * of the std::function is 'void', no value is assigned.
  */
-template <typename R, typename = std::enable_if_t<!std::is_void<R>::value>, typename... Args>
+template <typename R, typename = std::enable_if_t<!std::is_void<R>::value>,
+    typename = std::enable_if_t<!is_std_vector<R>::value>, typename... Args>
 void callAndAssignAnyRpcResult(std::function<R(Args...)> func, anyrpc::Value& result, Args... args)
 {
     result = func(std::forward<Args>(args)...);