Introduce memory mapping API

plugins/inmemoryscanner/Readme.md

@@ -68,8 +68,6 @@ For example the following diagram shows the memory padding of one VAD region con
 
 Shared memory regions that are not the base image of the process are skipped by default in order to reduce scanning time.
 This behavior can be controlled via the `scan_all_regions` config option.
-To further optimize scan duration, memory regions >50MB will be ignored as well.
-If desired, it is possible to increase or reduce the threshold via the `maximum_scan_size` config option.
 
 ### In Depth Example
 
@@ -86,7 +84,7 @@ Consider the following VAD entry from the vad tree of a process `winlogon.exe` w
 | `EndAddress`   | 1f43b599000  |
 
 This region has a size of `0x1f43b599000` - `0x1f43b593000` = `0x6000`.
-However the pages from `0x1f43b596000` to `0x1f43b598000` (size `0x2000`) are not mapped into memory.
+However, the pages from `0x1f43b596000` to `0x1f43b598000` (size `0x2000`) are not mapped into memory.
 Therefore, the resulting files will have the size of `0x5000` (mapped size + one zero-page). Note that the start and end address however are the original ones.
 
 ```console
@@ -127,7 +125,6 @@ For this, add the following parts to the _VMICore_ config and tweak them to your
 | `directory`         | Path to the folder where the compiled _VMICore_ plugins are located.                                                                                                       |
 | `dump_memory`       | Boolean. If set to `true` will result in scanned memory being dumped to files. Regions will be dumped to an `inmemorydumps` subfolder in the output directory.             |
 | `ignored_processes` | List with processes that will not be scanned (or dumped) during the final scan.                                                                                            |
-| `maximum_scan_size` | Number of bytes for the size of the largest contiguous memory region that will still be scanned. Defaults to `52428800` (50MB).                                            |
 | `output_path`       | Optional output path. If this is a relative path it is interpreted relatively to the _VMICore_ results directory.                                                          |
 | `plugins`           | Add your plugin here by the exact name of your shared library (e.g. `libinmemoryscanner.so`). All plugin specific config keys should be added as sub-keys under this name. |
 | `scan_all_regions`  | Optional boolean (defaults to `false`). Indicates whether to eagerly scan all memory regions as opposed to ignoring shared memory.                                         |
@@ -144,7 +141,6 @@ plugin_system:
       signature_file: /usr/local/share/inmemsigs/sigs.sig
       dump_memory: false
       scan_all_regions: false
-      maximum_scan_size: 52428800
       output_path: ""
       ignored_processes:
         - SearchUI.exe

plugins/inmemoryscanner/src/lib/Common.h

@@ -4,6 +4,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <vmicore/os/PagingDefinitions.h>
 
 namespace InMemoryScanner
 {
@@ -26,4 +27,9 @@ namespace InMemoryScanner
         std::string ruleNamespace;
         std::vector<Match> matches;
     };
+
+    inline std::size_t bytesToNumberOfPages(std::size_t size)
+    {
+        return (size + VmiCore::PagingDefinitions::pageSizeInBytes - 1) / VmiCore::PagingDefinitions::pageSizeInBytes;
+    }
 }

plugins/inmemoryscanner/src/lib/Config.cpp

@@ -8,8 +8,6 @@ using VmiCore::Plugin::PluginInterface;
 
 namespace InMemoryScanner
 {
-    constexpr uint64_t defaultMaxScanSize = 52428800; // 50MB
-
     Config::Config(const PluginInterface* pluginInterface) : pluginInterface(pluginInterface) {}
 
     void Config::parseConfiguration(const IPluginConfig& config)
@@ -19,7 +17,6 @@ namespace InMemoryScanner
         outputPath = rootNode["output_path"].as<std::string>();
         dumpMemory = rootNode["dump_memory"].as<bool>(false);
         scanAllRegions = rootNode["scan_all_regions"].as<bool>(false);
-        maximumScanSize = rootNode["maximum_scan_size"].as<uint64_t>(defaultMaxScanSize);
 
         auto ignoredProcessesVec =
             rootNode["ignored_processes"].as<std::vector<std::string>>(std::vector<std::string>());
@@ -58,11 +55,6 @@ namespace InMemoryScanner
         return dumpMemory;
     }
 
-    uint64_t Config::getMaximumScanSize() const
-    {
-        return maximumScanSize;
-    }
-
     void Config::overrideDumpMemoryFlag(bool value)
     {
         dumpMemory = value;

plugins/inmemoryscanner/src/lib/Config.h

@@ -31,8 +31,6 @@ namespace InMemoryScanner
 
         [[nodiscard]] virtual bool isDumpingMemoryActivated() const = 0;
 
-        [[nodiscard]] virtual uint64_t getMaximumScanSize() const = 0;
-
         virtual void overrideDumpMemoryFlag(bool value) = 0;
 
       protected:
@@ -58,8 +56,6 @@ namespace InMemoryScanner
 
         [[nodiscard]] bool isDumpingMemoryActivated() const override;
 
-        [[nodiscard]] uint64_t getMaximumScanSize() const override;
-
         void overrideDumpMemoryFlag(bool value) override;
 
       private:
@@ -69,6 +65,5 @@ namespace InMemoryScanner
         std::set<std::string> ignoredProcesses;
         bool dumpMemory{};
         bool scanAllRegions{};
-        uint64_t maximumScanSize{};
     };
 }

plugins/inmemoryscanner/src/lib/Scanner.cpp

@@ -1,12 +1,17 @@
 #include "Scanner.h"
+#include "Common.h"
 #include "Filenames.h"
 #include <future>
 #include <iterator>
+#include <span>
+#include <vmicore/os/PagingDefinitions.h>
 
 using VmiCore::ActiveProcessInformation;
 using VmiCore::addr_t;
+using VmiCore::MappedRegion;
 using VmiCore::MemoryRegion;
 using VmiCore::pid_t;
+using VmiCore::PagingDefinitions::pageSizeInBytes;
 using VmiCore::Plugin::LogLevel;
 using VmiCore::Plugin::PluginInterface;
 
@@ -51,8 +56,43 @@ namespace InMemoryScanner
         return verdict;
     }
 
-    void
-    Scanner::scanMemoryRegion(pid_t pid, const std::string& processName, const MemoryRegion& memoryRegionDescriptor)
+    std::vector<uint8_t> Scanner::constructPaddedMemoryRegion(const std::vector<MappedRegion>& regions)
+    {
+        std::vector<uint8_t> result;
+
+        if (regions.empty())
+        {
+            return result;
+        }
+
+        std::size_t regionSize = 0;
+        for (const auto& region : regions)
+        {
+            regionSize += region.mapping.size();
+            regionSize += pageSizeInBytes;
+        }
+        // last region should not have succeeding padding page
+        regionSize -= pageSizeInBytes;
+
+        result.reserve(regionSize);
+        // copy first region
+        std::copy(regions.front().mapping.begin(), regions.front().mapping.end(), std::back_inserter(result));
+
+        for (std::size_t i = 1; i < regions.size(); i++)
+        {
+            const auto& region = regions[i];
+            // padding page
+            result.insert(result.end(), pageSizeInBytes, 0);
+            std::copy(region.mapping.begin(), region.mapping.end(), std::back_inserter(result));
+        }
+
+        return result;
+    }
+
+    void Scanner::scanMemoryRegion(pid_t pid,
+                                   addr_t dtb,
+                                   const std::string& processName,
+                                   const MemoryRegion& memoryRegionDescriptor)
     {
         pluginInterface->logMessage(LogLevel::info,
                                     LOG_FILENAME,
@@ -62,24 +102,11 @@ namespace InMemoryScanner
 
         if (shouldRegionBeScanned(memoryRegionDescriptor))
         {
-            auto scanSize = memoryRegionDescriptor.size;
-            auto maximumScanSize = configuration->getMaximumScanSize();
-            if (scanSize > maximumScanSize)
-            {
-                pluginInterface->logMessage(
-                    LogLevel::info, LOG_FILENAME, "Memory region is too big, reduce to " + intToHex(maximumScanSize));
-                scanSize = maximumScanSize;
-            }
-
-            pluginInterface->logMessage(
-                LogLevel::debug, LOG_FILENAME, "Start getProcessMemoryRegion with size: " + intToHex(scanSize));
-
-            auto memoryRegion = pluginInterface->readProcessMemoryRegion(pid, memoryRegionDescriptor.base, scanSize);
+            auto memoryMapping = pluginInterface->mapProcessMemoryRegion(
+                memoryRegionDescriptor.base, dtb, bytesToNumberOfPages(memoryRegionDescriptor.size));
+            auto mappedRegions = memoryMapping->getMappedRegions().lock();
 
-            pluginInterface->logMessage(LogLevel::debug,
-                                        LOG_FILENAME,
-                                        "End getProcessMemoryRegion with size: " + intToHex(memoryRegion->size()));
-            if (memoryRegion->empty())
+            if (mappedRegions->empty())
             {
                 pluginInterface->logMessage(
                     LogLevel::debug, LOG_FILENAME, "Extracted memory region has size 0, skipping");
@@ -91,18 +118,22 @@ namespace InMemoryScanner
                     pluginInterface->logMessage(LogLevel::debug,
                                                 LOG_FILENAME,
                                                 "Start dumpVadRegionToFile with size: " +
-                                                    intToHex(memoryRegion->size()));
-                    dumping->dumpMemoryRegion(processName, pid, memoryRegionDescriptor, *memoryRegion);
+                                                    intToHex(memoryMapping->getSizeInGuest()));
+
+                    auto paddedRegion = constructPaddedMemoryRegion(*mappedRegions);
+
+                    dumping->dumpMemoryRegion(processName, pid, memoryRegionDescriptor, paddedRegion);
                     pluginInterface->logMessage(LogLevel::debug, LOG_FILENAME, "End dumpVadRegionToFile");
                 }
 
-                pluginInterface->logMessage(
-                    LogLevel::debug, LOG_FILENAME, "Start scanMemory with size: " + intToHex(memoryRegion->size()));
+                pluginInterface->logMessage(LogLevel::debug,
+                                            LOG_FILENAME,
+                                            "Start scanMemory with size: " + intToHex(memoryMapping->getSizeInGuest()));
 
                 // The semaphore protects the yara rules from being accessed more than YR_MAX_THREADS (32 atm.) times in
                 // parallel.
                 semaphore.wait();
-                auto results = yaraEngine->scanMemory(*memoryRegion);
+                auto results = yaraEngine->scanMemory(*mappedRegions);
                 semaphore.notify();
 
                 pluginInterface->logMessage(LogLevel::debug, LOG_FILENAME, "End scanMemory");
@@ -147,8 +178,10 @@ namespace InMemoryScanner
                 {
                     try
                     {
-                        scanMemoryRegion(
-                            processInformation->pid, *processInformation->fullName, memoryRegionDescriptor);
+                        scanMemoryRegion(processInformation->pid,
+                                         processInformation->processCR3,
+                                         *processInformation->fullName,
+                                         memoryRegionDescriptor);
                     }
                     catch (const std::exception& exc)
                     {

plugins/inmemoryscanner/src/lib/Scanner.h

@@ -40,7 +40,10 @@ namespace InMemoryScanner
 
         bool shouldRegionBeScanned(const VmiCore::MemoryRegion& memoryRegionDescriptor);
 
+        static std::vector<uint8_t> constructPaddedMemoryRegion(const std::vector<VmiCore::MappedRegion>& regions);
+
         void scanMemoryRegion(pid_t pid,
+                              uint64_t dtb,
                               const std::string& processName,
                               const VmiCore::MemoryRegion& memoryRegionDescriptor);
 

plugins/inmemoryscanner/src/lib/Yara.cpp

@@ -1,5 +1,7 @@
 #include "Yara.h"
 
+using VmiCore::MappedRegion;
+
 namespace InMemoryScanner
 {
     Yara::Yara(const std::string& rulesFile)
@@ -25,15 +27,18 @@ namespace InMemoryScanner
         yr_finalize();
     }
 
-    std::unique_ptr<std::vector<Rule>> Yara::scanMemory(std::vector<uint8_t>& buffer)
+    std::unique_ptr<std::vector<Rule>> Yara::scanMemory(const std::vector<MappedRegion>& mappedRegions)
     {
         auto results = std::make_unique<std::vector<Rule>>();
-        int err = 0;
 
-        err = yr_rules_scan_mem(rules, buffer.data(), buffer.size(), 0, yaraCallback, results.get(), 0);
-        if (err != ERROR_SUCCESS)
+        for (const auto& mappedRegion : mappedRegions)
         {
-            throw YaraException("Error scanning memory. Error code: " + std::to_string(err));
+            auto err = yr_rules_scan_mem(
+                rules, mappedRegion.mapping.data(), mappedRegion.mapping.size(), 0, yaraCallback, results.get(), 0);
+            if (err != ERROR_SUCCESS)
+            {
+                throw YaraException("Error scanning memory. Error code: " + std::to_string(err));
+            }
         }
 
         return results;

plugins/inmemoryscanner/src/lib/Yara.h

@@ -12,7 +12,7 @@ namespace InMemoryScanner
 
         ~Yara() override;
 
-        std::unique_ptr<std::vector<Rule>> scanMemory(std::vector<uint8_t>& buffer) override;
+        std::unique_ptr<std::vector<Rule>> scanMemory(const std::vector<VmiCore::MappedRegion>& mappedRegions) override;
 
       private:
         YR_RULES* rules = nullptr;

plugins/inmemoryscanner/src/lib/YaraInterface.h

@@ -2,6 +2,7 @@
 
 #include "Common.h"
 #include <memory>
+#include <vmicore/vmi/IMemoryMapping.h>
 
 namespace InMemoryScanner
 {
@@ -16,7 +17,8 @@ namespace InMemoryScanner
       public:
         virtual ~YaraInterface() = default;
 
-        virtual std::unique_ptr<std::vector<Rule>> scanMemory(std::vector<uint8_t>& buffer) = 0;
+        virtual std::unique_ptr<std::vector<Rule>>
+        scanMemory(const std::vector<VmiCore::MappedRegion>& mappedRegions) = 0;
 
       protected:
         YaraInterface() = default;

plugins/inmemoryscanner/test/FakeYara.cpp

@@ -4,7 +4,8 @@
 
 namespace InMemoryScanner
 {
-    std::unique_ptr<std::vector<Rule>> FakeYara::scanMemory([[maybe_unused]] std::vector<uint8_t>& buffer)
+    std::unique_ptr<std::vector<Rule>>
+    FakeYara::scanMemory([[maybe_unused]] const std::vector<VmiCore::MappedRegion>& mappedRegions)
     {
         concurrentThreads++;
         if (concurrentThreads > YR_MAX_THREADS)

plugins/inmemoryscanner/test/FakeYara.h

@@ -7,7 +7,7 @@ namespace InMemoryScanner
     class FakeYara : public YaraInterface
     {
       public:
-        std::unique_ptr<std::vector<Rule>> scanMemory(std::vector<uint8_t>& buffer) override;
+        std::unique_ptr<std::vector<Rule>> scanMemory(const std::vector<VmiCore::MappedRegion>& mappedRegions) override;
 
         bool max_threads_exceeded = false;