Update Yara to 4.0.1

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 # dev
 
+* Enhancement: YARA updated to version 4.0.1 ([#758](https://github.com/avast/retdec/issues/758)), fixed Mach-O parsing issue ([#283](https://github.com/avast/retdec/issues/283)).
 * Enhancement: Improved detection of many packers/installers/compilers in `retdec-fileinfo`, including Armadillo ([#733](https://github.com/avast/retdec/pull/733)), VMProtect ([#734](https://github.com/avast/retdec/pull/734)), Petite ([#735](https://github.com/avast/retdec/pull/735)), Enigma ([#741](https://github.com/avast/retdec/pull/741)), ASPack ([#743](https://github.com/avast/retdec/pull/743)), Eziriz ([#746](https://github.com/avast/retdec/pull/746)), PyInstaller ([#748](https://github.com/avast/retdec/pull/748)), Astrum InstallWizard ([#753](https://github.com/avast/retdec/pull/753)), AutoHotKey ([#756](https://github.com/avast/retdec/pull/756)), AutoIt ([#757](https://github.com/avast/retdec/pull/757)), BAT to PE-EXE script compilers ([#761](https://github.com/avast/retdec/pull/761)), Bero ([#764](https://github.com/avast/retdec/pull/764)).
 * Enhancement: Enable .NET module in RetDec's YARA ([#747](https://github.com/avast/retdec/issues/747)).
 * Fix: Fixed build on some systems by adding missing includes of `<limits>` into `retdec-fileinfo` ([#745](https://github.com/avast/retdec/pull/745)).

cmake/deps.cmake

@@ -45,13 +45,12 @@ set(OPENSSL_ARCHIVE_SHA256
     CACHE INTERNAL ""
 )
 
-# 3.11.0
 set(YARA_URL
-    "https://github.com/VirusTotal/yara/archive/b9f925bb4e2b998bd6bb2f2e3cc2087c62fdd5b9.zip"
+    "https://github.com/VirusTotal/yara/archive/v4.0.1.zip"
     CACHE INTERNAL "URL of Yara archive to use."
 )
 set(YARA_ARCHIVE_SHA256
-    "8ebec236ede6f20f27ad597452a02dcc54ac3c73f09fea8c93c3a6bf10fbc890"
+    "4dcc6907b8537b67b52a61aa76f01196a4cc8e8e9f5fb6e4dd835692c2370e83"
     CACHE INTERNAL ""
 )
 

deps/yara/patch.cmake

@@ -5,67 +5,104 @@ function(patch_vcxproj file)
     set(new_content "${content}")
 
     string(REPLACE
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;PROFILING_ENABLED</PreprocessorDefinitions>"
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;DOTNET_MODULE;MACHO_MODULE;USE_WINDOWS_PROC</PreprocessorDefinitions>"
+        "jansson.lib"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<AdditionalDependencies>jansson.lib;libcrypto.lib;%(AdditionalDependencies)</AdditionalDependencies>"
-        "<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>"
+        "libcrypto.lib"
+        ""
         new_content
         "${new_content}"
     )
+	# We also use this to enable macho module.
     string(REPLACE
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIBC;PROFILING_ENABLED</PreprocessorDefinitions>"
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;DOTNET_MODULE;MACHO_MODULE;USE_WINDOWS_PROC</PreprocessorDefinitions>"
+        "CUCKOO_MODULE"
+        "MACHO_MODULE"
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<AdditionalDependencies>crypt32.lib;ws2_32.lib;advapi32.lib;jansson.lib;libcrypto.lib;%(AdditionalDependencies)</AdditionalDependencies>"
-        "<AdditionalDependencies>crypt32.lib;ws2_32.lib;advapi32.lib;%(AdditionalDependencies)</AdditionalDependencies>"
+        "HASH_MODULE"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB</PreprocessorDefinitions>"
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;DOTNET_MODULE;MACHO_MODULE;USE_WINDOWS_PROC</PreprocessorDefinitions>"
+        "HAVE_LIBCRYPTO"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;NDEBUG=1</PreprocessorDefinitions>"
-        "<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;DOTNET_MODULE;MACHO_MODULE;USE_WINDOWS_PROC;NDEBUG=1</PreprocessorDefinitions>"
+        "PROFILING_ENABLED"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<AdditionalIncludeDirectories>..\\..\\..\\libyara;..\\..\\..\\libyara\\include;..\\..\\..;..\\packages\\YARA.Jansson.x64.1.1.0\\include;..\\packages\\YARA.OpenSSL.x64.1.1.0\\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>"
-        "<AdditionalIncludeDirectories>..\\..\\..\\libyara;..\\..\\..\\libyara\\include;..\\..\\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>"
+        "..\\packages\\YARA.Jansson.x64.1.1.0\\include"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<AdditionalLibraryDirectories>..\\packages\\YARA.OpenSSL.x64.1.1.0\\lib;..\\packages\\YARA.Jansson.x64.1.1.0\\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>"
-        "<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>"
+        "..\\packages\\YARA.Jansson.x86.1.1.0\\include"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.Jansson.x64.1.1.0\\lib"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.Jansson.x86.1.1.0\\lib"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.OpenSSL.x64.1.1.0\\include"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.OpenSSL.x86.1.1.0\\include"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.OpenSSL.x64.1.1.0\\lib"
+        ""
+        new_content
+        "${new_content}"
+    )
+    string(REPLACE
+        "..\\packages\\YARA.OpenSSL.x86.1.1.0\\lib"
+        ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<ClCompile Include=\"..\\..\\..\\libyara\\modules\\cuckoo.c\" />"
+		"<ClCompile Include=\"..\\..\\..\\libyara\\modules\\cuckoo\\cuckoo.c\" />"
         ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<ClCompile Include=\"..\\..\\..\\libyara\\modules\\dex.c\" />"
+        "<ClCompile Include=\"..\\..\\..\\libyara\\modules\\dex\\dex.c\" />"
         ""
         new_content
         "${new_content}"
     )
     string(REPLACE
-        "<ClCompile Include=\"..\\..\\..\\libyara\\modules\\hash.c\" />"
+        "<ClCompile Include=\"..\\..\\..\\libyara\\modules\\hash\\hash.c\" />"
         ""
         new_content
         "${new_content}"

include/retdec/yaracpp/yara_detector.h

@@ -0,0 +1,140 @@
+/**
+ * @file include/retdec/yaracpp/yara_detector.h
+ * @brief Interpret of YARA rules.
+ * @copyright (c) 2017 Avast Software, licensed under the MIT license
+ */
+
+#ifndef RETDEC_YARACPP_YARA_DETECTOR_H
+#define RETDEC_YARACPP_YARA_DETECTOR_H
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "retdec/yaracpp/yara_rule.h"
+
+typedef struct _YR_COMPILER YR_COMPILER;
+typedef struct YR_RULES YR_RULES;
+typedef struct YR_SCAN_CONTEXT YR_SCAN_CONTEXT;
+
+namespace retdec {
+namespace yaracpp {
+
+/**
+ * Interpret of YARA rules
+ */
+class YaraDetector
+{
+	public:
+		/**
+		 * Structure for callback function
+		 */
+		class CallbackSettings
+		{
+			private:
+				/// set to @c true if you want store all rules (not only detected)
+				bool storeAll;
+				/// link to detected rules
+				std::vector<YaraRule> &storedDetected;
+				/// link to undetected rules
+				std::vector<YaraRule> &storedUndetected;
+			public:
+				CallbackSettings(
+						bool cStoreAll,
+						std::vector<YaraRule> &cDetected,
+						std::vector<YaraRule> &cUndetected
+				);
+
+				/// @name Other methods
+				/// @{
+				void addDetected(YaraRule &rule);
+				void addUndetected(YaraRule &rule);
+				bool storeAllRules() const;
+				/// @}
+		};
+
+		struct RuleFile
+		{
+			RuleFile(
+					const std::string& pathToFile_,
+					bool precompiled_,
+					FILE* handle_)
+					: pathToFile(pathToFile_)
+					, precompiled(precompiled_)
+					, handle(handle_)
+			{}
+
+			std::string pathToFile;
+			bool precompiled;
+			FILE* handle;
+		};
+
+	private:
+		/// compiler or text rules
+		YR_COMPILER *compiler = nullptr;
+		/// representation of files with rules
+		std::vector<FILE*> files;
+		/// representation of detected rules
+		std::vector<YaraRule> detectedRules;
+		/// representation of undetected rules
+		std::vector<YaraRule> undetectedRules;
+		/// rules from input text files
+		YR_RULES* textFilesRules = nullptr;
+		/// rules from precompiled files
+		std::vector<YR_RULES*> precompiledRules;
+		/// internal state of instance
+		bool stateIsValid = true;
+		/// indicates whether text files need recompilation
+		bool needsRecompilation = true;
+
+		/// @name Static auxiliary methods
+		/// @{
+		static int yaraCallback(
+				YR_SCAN_CONTEXT* context,
+				int message,
+				void *messageData,
+				void *userData
+		);
+		/// @}
+
+		/// @name Auxiliary detection methods
+		/// @{
+		template <typename T> bool analyzeWithScan(
+				T&& value,
+				bool storeAllRules = false
+		);
+		YR_RULES* getCompiledRules();
+		/// @}
+	public:
+		YaraDetector();
+		~YaraDetector();
+
+		/// @name Other methods
+		/// @{
+		bool addRules(const char *string);
+		bool addRuleFile(
+				const std::string &pathToFile,
+				const std::string &nameSpace = std::string()
+		);
+		bool isInValidState() const;
+		/// @}
+
+		/// @name Detection methods
+		/// @{
+		bool analyze(
+				const std::string &pathToInputFile,
+				bool storeAllRules = false
+		);
+		bool analyze(
+				std::vector<std::uint8_t> &bytes,
+				bool storeAllRules = false
+		);
+		const std::vector<YaraRule>& getDetectedRules() const;
+		const std::vector<YaraRule>& getUndetectedRules() const;
+		/// @}
+};
+
+} // namespace yaracpp
+} // namespace retdec
+
+#endif