Introduce memory mapping API

plugins/inmemoryscanner/Readme.md

@@ -71,8 +71,6 @@ For example the following diagram shows the memory padding of one VAD region con
 
 Shared memory regions that are not the base image of the process are skipped by default in order to reduce scanning time.
 This behavior can be controlled via the `scan_all_regions` config option.
-To further optimize scan duration, memory regions >50MB will be ignored as well.
-If desired, it is possible to increase or reduce the threshold via the `maximum_scan_size` config option.
 
 ### In Depth Example
 
@@ -89,7 +87,7 @@ Consider the following VAD entry from the vad tree of a process `winlogon.exe` w
 | `EndAddress`   | 1f43b599000  |
 
 This region has a size of `0x1f43b599000` - `0x1f43b593000` = `0x6000`.
-However the pages from `0x1f43b596000` to `0x1f43b598000` (size `0x2000`) are not mapped into memory.
+However, the pages from `0x1f43b596000` to `0x1f43b598000` (size `0x2000`) are not mapped into memory.
 Therefore, the resulting files will have the size of `0x5000` (mapped size + one zero-page). Note that the start and end address however are the original ones.
 
 ```console
@@ -130,7 +128,6 @@ For this, add the following parts to the _VMICore_ config and tweak them to your
 | `directory`         | Path to the folder where the compiled _VMICore_ plugins are located.                                                                                                       |
 | `dump_memory`       | Boolean. If set to `true` will result in scanned memory being dumped to files. Regions will be dumped to an `inmemorydumps` subfolder in the output directory.             |
 | `ignored_processes` | List with processes that will not be scanned (or dumped) during the final scan.                                                                                            |
-| `maximum_scan_size` | Number of bytes for the size of the largest contiguous memory region that will still be scanned. Defaults to `52428800` (50MB).                                            |
 | `output_path`       | Optional output path. If this is a relative path it is interpreted relatively to the _VMICore_ results directory.                                                          |
 | `plugins`           | Add your plugin here by the exact name of your shared library (e.g. `libinmemoryscanner.so`). All plugin specific config keys should be added as sub-keys under this name. |
 | `scan_all_regions`  | Optional boolean (defaults to `false`). Indicates whether to eagerly scan all memory regions as opposed to ignoring shared memory.                                         |
@@ -147,7 +144,6 @@ plugin_system:
       signature_file: /usr/local/share/inmemsigs/sigs.sig
       dump_memory: false
       scan_all_regions: false
-      maximum_scan_size: 52428800
       output_path: ""
       ignored_processes:
         - SearchUI.exe

plugins/inmemoryscanner/src/lib/CMakeLists.txt

@@ -7,7 +7,7 @@ add_library(inmemoryscanner-obj OBJECT
         InMemory.cpp
         OutputXML.cpp
         Scanner.cpp
-        Yara.cpp)
+        YaraInterface.cpp)
 target_compile_features(inmemoryscanner-obj PUBLIC cxx_std_20)
 set_target_properties(inmemoryscanner-obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 target_include_directories(inmemoryscanner-obj INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
@@ -16,6 +16,11 @@ include(FindPkgConfig)
 
 pkg_check_modules(YARA REQUIRED yara>=4)
 target_link_libraries(inmemoryscanner-obj PUBLIC ${YARA_LINK_LIBRARIES})
+
+if (${YARA_VERSION} VERSION_GREATER_EQUAL 4.1)
+    target_compile_definitions(inmemoryscanner-obj PRIVATE LIBYARA_4_1)
+endif ()
+
 pkg_check_modules(TCLAP REQUIRED tclap>=1.2)
 
 include(FetchContent)

plugins/inmemoryscanner/src/lib/Common.h

@@ -5,6 +5,7 @@
 #include <string>
 #include <vector>
 #include <vmicore/filename.h>
+#include <vmicore/os/PagingDefinitions.h>
 
 #define INMEMORY_LOGGER_NAME std::string("InMemory_").append(FILENAME_STEM)
 
@@ -14,12 +15,21 @@ namespace InMemoryScanner
     {
         std::string matchName;
         int64_t position;
+
+        bool operator==(const Match& rhs) const = default;
     };
 
     struct Rule
     {
         std::string ruleName;
         std::string ruleNamespace;
         std::vector<Match> matches;
+
+        bool operator==(const Rule& rhs) const = default;
     };
+
+    inline std::size_t bytesToNumberOfPages(std::size_t size)
+    {
+        return (size + VmiCore::PagingDefinitions::pageSizeInBytes - 1) / VmiCore::PagingDefinitions::pageSizeInBytes;
+    }
 }

plugins/inmemoryscanner/src/lib/Config.cpp

@@ -8,8 +8,6 @@ using VmiCore::Plugin::PluginInterface;
 
 namespace InMemoryScanner
 {
-    constexpr uint64_t defaultMaxScanSize = 52428800; // 50MB
-
     Config::Config(const PluginInterface* pluginInterface)
         : logger(pluginInterface->newNamedLogger(INMEMORY_LOGGER_NAME))
     {
@@ -23,7 +21,6 @@ namespace InMemoryScanner
         outputPath = rootNode["output_path"].as<std::string>();
         dumpMemory = rootNode["dump_memory"].as<bool>(false);
         scanAllRegions = rootNode["scan_all_regions"].as<bool>(false);
-        maximumScanSize = rootNode["maximum_scan_size"].as<uint64_t>(defaultMaxScanSize);
 
         auto ignoredProcessesVec =
             rootNode["ignored_processes"].as<std::vector<std::string>>(std::vector<std::string>());
@@ -61,11 +58,6 @@ namespace InMemoryScanner
         return dumpMemory;
     }
 
-    uint64_t Config::getMaximumScanSize() const
-    {
-        return maximumScanSize;
-    }
-
     void Config::overrideDumpMemoryFlag(bool value)
     {
         dumpMemory = value;

plugins/inmemoryscanner/src/lib/Config.h

@@ -32,8 +32,6 @@ namespace InMemoryScanner
 
         [[nodiscard]] virtual bool isDumpingMemoryActivated() const = 0;
 
-        [[nodiscard]] virtual uint64_t getMaximumScanSize() const = 0;
-
         virtual void overrideDumpMemoryFlag(bool value) = 0;
 
       protected:
@@ -59,8 +57,6 @@ namespace InMemoryScanner
 
         [[nodiscard]] bool isDumpingMemoryActivated() const override;
 
-        [[nodiscard]] uint64_t getMaximumScanSize() const override;
-
         void overrideDumpMemoryFlag(bool value) override;
 
       private:
@@ -70,6 +66,5 @@ namespace InMemoryScanner
         std::set<std::string> ignoredProcesses;
         bool dumpMemory{};
         bool scanAllRegions{};
-        uint64_t maximumScanSize{};
     };
 }

plugins/inmemoryscanner/src/lib/IYaraInterface.h

@@ -0,0 +1,30 @@
+#ifndef INMEMORYSCANNER_IYARAINTERFACE_H
+#define INMEMORYSCANNER_IYARAINTERFACE_H
+
+#include "Common.h"
+#include <span>
+#include <stdexcept>
+#include <vector>
+#include <vmicore/vmi/IMemoryMapping.h>
+
+namespace InMemoryScanner
+{
+    class YaraException : public std::runtime_error
+    {
+        using std::runtime_error::runtime_error;
+    };
+
+    class IYaraInterface
+    {
+      public:
+        virtual ~IYaraInterface() = default;
+
+        virtual std::vector<Rule> scanMemory(VmiCore::addr_t regionBase,
+                                             std::span<const VmiCore::MappedRegion> mappedRegions) = 0;
+
+      protected:
+        IYaraInterface() = default;
+    };
+}
+
+#endif // INMEMORYSCANNER_IYARAINTERFACE_H

plugins/inmemoryscanner/src/lib/InMemory.cpp

@@ -11,7 +11,7 @@
 #include "Config.h"
 #include "Dumping.h"
 #include "Filenames.h"
-#include "Yara.h"
+#include "YaraInterface.h"
 #include <memory>
 #include <string>
 #include <tclap/CmdLine.h>
@@ -44,7 +44,7 @@ namespace InMemoryScanner
         {
             configuration->overrideDumpMemoryFlag(dumpMemoryArgument.getValue());
         }
-        auto yara = std::make_unique<Yara>(configuration->getSignatureFile());
+        auto yara = std::make_unique<YaraInterface>(configuration->getSignatureFile());
         auto dumping = std::make_unique<Dumping>(pluginInterface, configuration);
         scanner = std::make_unique<Scanner>(pluginInterface, configuration, std::move(yara), std::move(dumping));
     }

plugins/inmemoryscanner/src/lib/Scanner.cpp

@@ -1,26 +1,29 @@
 #include "Scanner.h"
 #include "Common.h"
 #include "Filenames.h"
+#include <algorithm>
 #include <fmt/core.h>
 #include <future>
-#include <iterator>
 #include <vmicore/callback.h>
+#include <vmicore/os/PagingDefinitions.h>
 
 using VmiCore::ActiveProcessInformation;
 using VmiCore::addr_t;
+using VmiCore::MappedRegion;
 using VmiCore::MemoryRegion;
 using VmiCore::pid_t;
+using VmiCore::PagingDefinitions::pageSizeInBytes;
 using VmiCore::Plugin::PluginInterface;
 
 namespace InMemoryScanner
 {
     Scanner::Scanner(PluginInterface* pluginInterface,
                      std::shared_ptr<IConfig> configuration,
-                     std::unique_ptr<YaraInterface> yaraEngine,
+                     std::unique_ptr<IYaraInterface> yaraInterface,
                      std::unique_ptr<IDumping> dumping)
         : pluginInterface(pluginInterface),
           configuration(std::move(configuration)),
-          yaraEngine(std::move(yaraEngine)),
+          yaraInterface(std::move(yaraInterface)),
           dumping(std::move(dumping)),
           logger(pluginInterface->newNamedLogger(INMEMORY_LOGGER_NAME)),
           inMemResultsLogger(pluginInterface->newNamedLogger(INMEMORY_LOGGER_NAME))
@@ -45,76 +48,104 @@ namespace InMemoryScanner
 
     bool Scanner::shouldRegionBeScanned(const MemoryRegion& memoryRegionDescriptor)
     {
-        bool verdict = true;
         if (configuration->isScanAllRegionsActivated())
         {
             return true;
         }
         if (memoryRegionDescriptor.isSharedMemory && !memoryRegionDescriptor.isProcessBaseImage)
         {
-            verdict = false;
             logger->info("Skipping: Is shared memory and not the process base image.");
+            return false;
         }
-        return verdict;
+        return true;
     }
 
-    void
-    Scanner::scanMemoryRegion(pid_t pid, const std::string& processName, const MemoryRegion& memoryRegionDescriptor)
+    std::vector<uint8_t> Scanner::constructPaddedMemoryRegion(std::span<const MappedRegion> regions)
+    {
+        std::vector<uint8_t> result;
+
+        if (regions.empty())
+        {
+            return result;
+        }
+
+        std::size_t regionSize = 0;
+        for (const auto& region : regions)
+        {
+            regionSize += region.asSpan().size();
+            regionSize += pageSizeInBytes;
+        }
+        // last region should not have succeeding padding page
+        regionSize -= pageSizeInBytes;
+
+        result.reserve(regionSize);
+        // copy first region
+        auto frontRegionSpan = regions.front().asSpan();
+        std::ranges::copy(frontRegionSpan.begin(), frontRegionSpan.end(), std::back_inserter(result));
+
+        // copy the rest of the regions with a padding page in between each chunk
+        for (std::size_t i = 1; i < regions.size(); i++)
+        {
+            result.insert(result.end(), pageSizeInBytes, 0);
+            auto regionSpan = regions[i].asSpan();
+            std::ranges::copy(regionSpan.begin(), regionSpan.end(), std::back_inserter(result));
+        }
+
+        return result;
+    }
+
+    void Scanner::scanMemoryRegion(pid_t pid,
+                                   addr_t dtb,
+                                   const std::string& processName,
+                                   const MemoryRegion& memoryRegionDescriptor)
     {
         logger->info("Scanning Memory region",
                      {{"VA", fmt::format("{:x}", memoryRegionDescriptor.base)},
                       {"Size", memoryRegionDescriptor.size},
                       {"Module", memoryRegionDescriptor.moduleName}});
 
-        if (shouldRegionBeScanned(memoryRegionDescriptor))
+        if (!shouldRegionBeScanned(memoryRegionDescriptor))
         {
-            auto scanSize = memoryRegionDescriptor.size;
-            auto maximumScanSize = configuration->getMaximumScanSize();
-            if (scanSize > maximumScanSize)
-            {
-                logger->info("Memory region is too big, reducing to maximum scan size",
-                             {{"Size", scanSize}, {"MaximumScanSize", maximumScanSize}});
-                scanSize = maximumScanSize;
-            }
+            return;
+        }
 
-            logger->debug("Start getProcessMemoryRegion", {{"Size", scanSize}});
+        auto memoryMapping = pluginInterface->mapProcessMemoryRegion(
+            memoryRegionDescriptor.base, dtb, bytesToNumberOfPages(memoryRegionDescriptor.size));
+        auto mappedRegions = memoryMapping->getMappedRegions();
 
-            auto memoryRegion = pluginInterface->readProcessMemoryRegion(pid, memoryRegionDescriptor.base, scanSize);
+        if (mappedRegions.empty())
+        {
+            logger->debug("Extracted memory region has size 0, skipping");
+            return;
+        }
 
-            logger->debug("End getProcessMemoryRegion", {{"Size", scanSize}});
-            if (memoryRegion->empty())
-            {
-                logger->debug("Extracted memory region has size 0, skipping");
-            }
-            else
-            {
-                if (configuration->isDumpingMemoryActivated())
-                {
-                    logger->debug("Start dumpVadRegionToFile", {{"Size", memoryRegion->size()}});
-                    dumping->dumpMemoryRegion(processName, pid, memoryRegionDescriptor, *memoryRegion);
-                    logger->debug("End dumpVadRegionToFile");
-                }
+        if (configuration->isDumpingMemoryActivated())
+        {
+            logger->debug("Start dumpVadRegionToFile", {{"Size", memoryRegionDescriptor.size}});
 
-                logger->debug("Start scanMemory", {{"Size", memoryRegion->size()}});
+            auto paddedRegion = constructPaddedMemoryRegion(mappedRegions);
 
-                // The semaphore protects the yara rules from being accessed more than YR_MAX_THREADS (32 atm.) times in
-                // parallel.
-                semaphore.wait();
-                auto results = yaraEngine->scanMemory(*memoryRegion);
-                semaphore.notify();
+            dumping->dumpMemoryRegion(processName, pid, memoryRegionDescriptor, paddedRegion);
+        }
 
-                logger->debug("End scanMemory");
+        logger->debug("Start scanMemory", {{"Size", memoryRegionDescriptor.size}});
 
-                if (!results->empty())
-                {
-                    for (const auto& result : *results)
-                    {
-                        pluginInterface->sendInMemDetectionEvent(result.ruleName);
-                    }
-                    outputXml.addResult(processName, pid, memoryRegionDescriptor.base, *results);
-                    logInMemoryResultToTextFile(processName, pid, memoryRegionDescriptor.base, *results);
-                }
+        // The semaphore protects the yara rules from being accessed more than YR_MAX_THREADS (32 atm.) times in
+        // parallel.
+        semaphore.wait();
+        auto results = yaraInterface->scanMemory(memoryRegionDescriptor.base, mappedRegions);
+        semaphore.notify();
+
+        logger->debug("End scanMemory");
+
+        if (!results.empty())
+        {
+            for (const auto& result : results)
+            {
+                pluginInterface->sendInMemDetectionEvent(result.ruleName);
             }
+            outputXml.addResult(processName, pid, memoryRegionDescriptor.base, results);
+            logInMemoryResultToTextFile(processName, pid, memoryRegionDescriptor.base, results);
         }
     }
 
@@ -141,8 +172,10 @@ namespace InMemoryScanner
                 {
                     try
                     {
-                        scanMemoryRegion(
-                            processInformation->pid, *processInformation->fullName, memoryRegionDescriptor);
+                        scanMemoryRegion(processInformation->pid,
+                                         processInformation->processUserDtb,
+                                         *processInformation->fullName,
+                                         memoryRegionDescriptor);
                     }
                     catch (const std::exception& exc)
                     {