diff --git a/.travis.yml b/.travis.yml index 2b70e1f..7a34293 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,15 @@ language: cpp sudo: required -dist: xenial +dist: focal matrix: include: - os: linux compiler: gcc - addons: &linux-gcc-5 { apt: { + addons: &linux-gcc-10 { apt: { sources: [ ubuntu-toolchain-r-test ], - packages: [ g++-5, cmake, qt5-default, qtdeclarative5-dev ] } } - env: MATRIX_EVAL="CXX=g++-5" BUILD_TYPE=Release + packages: [ g++-10, cmake, qt5-default, qtdeclarative5-dev ] } } + env: MATRIX_EVAL="CXX=g++-10" BUILD_TYPE=Release - os: linux compiler: gcc diff --git a/README.md b/README.md index 904c151..2460be5 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,33 @@ resultText = s.getStringProperty("mainWindow/results", "text") s.quit() ``` +You can also use [PyAutoGUI](https://pyautogui.readthedocs.io) in combination with +Spix. Have a look at the [example script](examples/RemoteCtrl/script/autogui.py). + ## What are the applications of Spix? -The obvious use for Spix is to automatically test the GUI of your Qt/QML application +The main use for Spix is to automatically test the UI of your Qt/QML application and make sure that it behaves as you expect. However, you can also use Spix as an easy way to remote control existing Qt/QML applications or to automatically generate and update screenshots for your documentation. +## Two modes of operation +Spix can be used in two ways, which are different in how events are generated and sent +to your application: + +### Generate Qt events directly +You can use Spix to directly create Qt events, either from C++ as a unit test, or from +an external script via the network through RPC. Since the Qt events are generated directly inside the +app, and do not come from the system, the mouse coursor will not actually move and interaction +with other applications is limited. On the plus side, this mechanism is independent from +the system your app is running on and can easily be used to control software on an embedded +device via the network (RPC). + +### Generate system events externally +In this case, Spix is not generating the events itself. Instead, you use a script to query +Spix for the screen coordinates of qt objects and then generate events on the system level +through other tools. One option is to use python together with PyAutoGUI for this, as is +done in the [RemoteCtrl](examples/RemoteCtrl) example. + # Requirements * Qt * AnyRPC diff --git a/examples/RemoteCtrl/script/autogui.py b/examples/RemoteCtrl/script/autogui.py new file mode 100644 index 0000000..7837a0d --- /dev/null +++ b/examples/RemoteCtrl/script/autogui.py @@ -0,0 +1,36 @@ +import xmlrpc.client +import time +import pyautogui # you might have to first `pip install pyautogui` + + +def clickItem(path): + # Query spix to get the location of the item + bounds = s.getBoundingBox(path) + # bounds = [x, y, width, height] as list + center_x = bounds[0] + bounds[2] / 2 + center_y = bounds[1] + bounds[3] / 2 + + # use pyautogui to move the cursor... + pyautogui.moveTo(center_x, center_y, duration=1) + # ...and click + pyautogui.click() + + + +# Connect to the rpc server running in the qt app +s = xmlrpc.client.ServerProxy('http://localhost:9000') + +# Wait a bit so that the user has time to move the app to the foreground +print("Make sure the qt app is in the foreground... waiting 3s...") +time.sleep(3) + +# Click buttons +clickItem("mainWindow/Button_1") +clickItem("mainWindow/Button_2") +clickItem("mainWindow/Button_2") +clickItem("mainWindow/Button_1") +clickItem("mainWindow/Button_2") + +# Query the qt app for the text contents of the results box +resultText = s.getStringProperty("mainWindow/results", "text") +print("Result:\n{}".format(resultText)) diff --git a/examples/RemoteCtrl/script/autogui_requirements.txt b/examples/RemoteCtrl/script/autogui_requirements.txt new file mode 100644 index 0000000..731c3d1 --- /dev/null +++ b/examples/RemoteCtrl/script/autogui_requirements.txt @@ -0,0 +1 @@ +PyAutoGUI==0.9.52 diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 50dc0cf..643a4c8 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -47,6 +47,8 @@ set(SOURCES src/Commands/EnterKey.h src/Commands/ExistsAndVisible.cpp src/Commands/ExistsAndVisible.h + src/Commands/GetBoundingBox.cpp + src/Commands/GetBoundingBox.h src/Commands/GetProperty.cpp src/Commands/GetProperty.h src/Commands/GetTestStatus.cpp diff --git a/lib/include/Spix/Data/Geometry.h b/lib/include/Spix/Data/Geometry.h index 0539160..4d9d253 100644 --- a/lib/include/Spix/Data/Geometry.h +++ b/lib/include/Spix/Data/Geometry.h @@ -11,6 +11,7 @@ namespace spix { using Real = double; struct Size { + Size(); Size(Real width, Real height); Real width; @@ -18,10 +19,19 @@ struct Size { }; struct Point { + Point(); Point(Real x, Real y); Real x; Real y; }; +struct Rect { + Rect(); + Rect(Real x, Real y, Real width, Real height); + + Point topLeft; + Size size; +}; + } // namespace spix diff --git a/lib/include/Spix/TestServer.h b/lib/include/Spix/TestServer.h index 3ec1eca..63038ca 100644 --- a/lib/include/Spix/TestServer.h +++ b/lib/include/Spix/TestServer.h @@ -11,6 +11,7 @@ #include #include +#include #include namespace spix { @@ -52,6 +53,7 @@ class TestServer { std::string getStringProperty(ItemPath path, std::string propertyName); void setStringProperty(ItemPath path, std::string propertyName, std::string propertyValue); + Rect getBoundingBox(ItemPath path); bool existsAndVisible(ItemPath path); std::vector getErrors(); diff --git a/lib/src/AnyRpcServer.cpp b/lib/src/AnyRpcServer.cpp index 364fea0..735365c 100644 --- a/lib/src/AnyRpcServer.cpp +++ b/lib/src/AnyRpcServer.cpp @@ -51,6 +51,11 @@ AnyRpcServer::AnyRpcServer(int anyrpcPort) [this](std::string path, std::string property, std::string value) { setStringProperty(std::move(path), std::move(property), std::move(value)); }); + utils::AddFunctionToAnyRpc(std::string)>(methodManager, "getBoundingBox", + "Return the bounding box of an item in screen coordinates", [this](std::string path) { + auto bounds = getBoundingBox(std::move(path)); + return std::vector {bounds.topLeft.x, bounds.topLeft.y, bounds.size.width, bounds.size.height}; + }); utils::AddFunctionToAnyRpc(methodManager, "existsAndVisible", "Returns true if the given object exists", [this](std::string path) { return existsAndVisible(std::move(path)); }); diff --git a/lib/src/Commands/GetBoundingBox.cpp b/lib/src/Commands/GetBoundingBox.cpp new file mode 100644 index 0000000..b0b09a5 --- /dev/null +++ b/lib/src/Commands/GetBoundingBox.cpp @@ -0,0 +1,34 @@ +/*** + * Copyright (C) Falko Axmann. All rights reserved. + * Licensed under the MIT license. + * See LICENSE.txt file in the project root for full license information. + ****/ + +#include "GetBoundingBox.h" + +#include + +namespace spix { +namespace cmd { + +GetBoundingBox::GetBoundingBox(ItemPath path, std::promise promise) +: m_path(std::move(path)) +, m_promise(std::move(promise)) +{ +} + +void GetBoundingBox::execute(CommandEnvironment& env) +{ + auto item = env.scene().itemAtPath(m_path); + + if (item) { + auto value = item->bounds(); + m_promise.set_value(value); + } else { + m_promise.set_value(Rect {0.0, 0.0, 0.0, 0.0}); + env.state().reportError("GetBoundingBox: Item not found: " + m_path.string()); + } +} + +} // namespace cmd +} // namespace spix diff --git a/lib/src/Commands/GetBoundingBox.h b/lib/src/Commands/GetBoundingBox.h new file mode 100644 index 0000000..4f1c9bf --- /dev/null +++ b/lib/src/Commands/GetBoundingBox.h @@ -0,0 +1,30 @@ +/*** + * Copyright (C) Falko Axmann. All rights reserved. + * Licensed under the MIT license. + * See LICENSE.txt file in the project root for full license information. + ****/ + +#pragma once + +#include "Command.h" +#include +#include + +#include + +namespace spix { +namespace cmd { + +class GetBoundingBox : public Command { +public: + GetBoundingBox(ItemPath path, std::promise promise); + + void execute(CommandEnvironment& env) override; + +private: + ItemPath m_path; + std::promise m_promise; +}; + +} // namespace cmd +} // namespace spix diff --git a/lib/src/Data/Geometry.cpp b/lib/src/Data/Geometry.cpp index a96cdcf..ee22fff 100644 --- a/lib/src/Data/Geometry.cpp +++ b/lib/src/Data/Geometry.cpp @@ -8,16 +8,38 @@ namespace spix { +Size::Size() +: width(0.0) +, height(0.0) +{ +} + Size::Size(Real width, Real height) : width(width) , height(height) { } +Point::Point() +: x(0.0) +, y(0.0) +{ +} + Point::Point(Real x, Real y) : x(x) , y(y) { } +Rect::Rect() +{ +} + +Rect::Rect(Real x, Real y, Real width, Real height) +: topLeft(x, y) +, size(width, height) +{ +} + } // namespace spix diff --git a/lib/src/Scene/Item.h b/lib/src/Scene/Item.h index d5df59a..c76779d 100644 --- a/lib/src/Scene/Item.h +++ b/lib/src/Scene/Item.h @@ -25,6 +25,8 @@ class Item { // Item properties virtual Size size() const = 0; + virtual Point position() const = 0; + virtual Rect bounds() const = 0; virtual std::string stringProperty(const std::string& name) const = 0; virtual void setStringProperty(const std::string& name, const std::string& value) = 0; virtual bool visible() const = 0; diff --git a/lib/src/Scene/Mock/MockItem.cpp b/lib/src/Scene/Mock/MockItem.cpp index d59d89e..e7a266d 100644 --- a/lib/src/Scene/Mock/MockItem.cpp +++ b/lib/src/Scene/Mock/MockItem.cpp @@ -18,6 +18,19 @@ Size MockItem::size() const return m_size; } +Point MockItem::position() const +{ + return Point {0.0, 0.0}; +} + +Rect MockItem::bounds() const +{ + Rect bounds {0.0, 0.0, 0.0, 0.0}; + bounds.topLeft = position(); + bounds.size = size(); + return bounds; +} + std::string MockItem::stringProperty(const std::string& name) const { return m_stringProperties.at(name); diff --git a/lib/src/Scene/Mock/MockItem.h b/lib/src/Scene/Mock/MockItem.h index 82bff2a..3d8e68e 100644 --- a/lib/src/Scene/Mock/MockItem.h +++ b/lib/src/Scene/Mock/MockItem.h @@ -18,6 +18,8 @@ class MockItem : public Item { // Item interface Size size() const override; + Point position() const override; + Rect bounds() const override; std::string stringProperty(const std::string& name) const override; void setStringProperty(const std::string& name, const std::string& value) override; bool visible() const override; diff --git a/lib/src/Scene/Qt/QtItem.cpp b/lib/src/Scene/Qt/QtItem.cpp index 5308f18..cd364aa 100644 --- a/lib/src/Scene/Qt/QtItem.cpp +++ b/lib/src/Scene/Qt/QtItem.cpp @@ -20,6 +20,24 @@ Size QtItem::size() const return Size {m_item->width(), m_item->height()}; } +Point QtItem::position() const +{ + // the point (0, 0) in item coordinates... + QPointF localPoint {0.0, 0.0}; + // ...is mapped to global to get the item position on screen + auto globalPoint = m_item->mapToGlobal(localPoint); + + return Point {globalPoint.rx(), globalPoint.ry()}; +} + +Rect QtItem::bounds() const +{ + Rect bounds {0.0, 0.0, 0.0, 0.0}; + bounds.topLeft = position(); + bounds.size = size(); + return bounds; +} + std::string QtItem::stringProperty(const std::string& name) const { auto value = m_item->property(name.c_str()); diff --git a/lib/src/Scene/Qt/QtItem.h b/lib/src/Scene/Qt/QtItem.h index bf3a5be..134e1a2 100644 --- a/lib/src/Scene/Qt/QtItem.h +++ b/lib/src/Scene/Qt/QtItem.h @@ -18,6 +18,8 @@ class QtItem : public Item { QtItem(QQuickItem* item); Size size() const override; + Point position() const override; + Rect bounds() const override; std::string stringProperty(const std::string& name) const override; void setStringProperty(const std::string& name, const std::string& value) override; bool visible() const override; diff --git a/lib/src/TestServer.cpp b/lib/src/TestServer.cpp index 31f83eb..c2f9ec9 100644 --- a/lib/src/TestServer.cpp +++ b/lib/src/TestServer.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -111,6 +112,16 @@ void TestServer::setStringProperty(ItemPath path, std::string propertyName, std: m_cmdExec->enqueueCommand(path, std::move(propertyName), std::move(propertyValue)); } +Rect TestServer::getBoundingBox(ItemPath path) +{ + std::promise promise; + auto result = promise.get_future(); + auto cmd = std::make_unique(path, std::move(promise)); + m_cmdExec->enqueueCommand(std::move(cmd)); + + return result.get(); +} + bool TestServer::existsAndVisible(ItemPath path) { std::promise promise; diff --git a/lib/src/Utils/AnyRpcUtils.h b/lib/src/Utils/AnyRpcUtils.h index 6cd6042..8af6cd6 100644 --- a/lib/src/Utils/AnyRpcUtils.h +++ b/lib/src/Utils/AnyRpcUtils.h @@ -18,6 +18,19 @@ namespace spix { namespace utils { +/** + * Generic type traits + */ +template +struct is_std_vector { + static const bool value = false; +}; + +template +struct is_std_vector> { + static const bool value = true; +}; + /** * Tools to convert an anyrpc::Value to a * particular type. If the requested type does not @@ -79,7 +92,8 @@ std::vector unpackAnyRpcParam(anyrpc::Value& value) * value to the given anyrpc::Value. If the return type * of the std::function is 'void', no value is assigned. */ -template ::value>, typename... Args> +template ::value>, + typename = std::enable_if_t::value>, typename... Args> void callAndAssignAnyRpcResult(std::function func, anyrpc::Value& result, Args... args) { result = func(std::forward(args)...);